Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study

Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study View ORCID Profile Anthony Papale , Robert Flattau , Nandan Vithlani , Deepti Mahajan , Yonah Ziemba , Tiffany Zavadsky , Anthony Carvino , View ORCID Profile Daniel King , Sandeep Nadella doi: https://doi.org/10.1101/2025.07.15.25331413 Anthony Papale 1 Northwell Health , New Hyde Park, New York MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Anthony Papale Robert Flattau 2 Zucker School of Medicine at Hofstra University , Hempstead, New York MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Nandan Vithlani 2 Zucker School of Medicine at Hofstra University , Hempstead, New York BS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Deepti Mahajan 1 Northwell Health , New Hyde Park, New York MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yonah Ziemba 1 Northwell Health , New Hyde Park, New York MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tiffany Zavadsky 1 Northwell Health , New Hyde Park, New York NP Find this author on Google Scholar Find this author on PubMed Search for this author on this site Anthony Carvino 1 Northwell Health , New Hyde Park, New York BS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Daniel King 1 Northwell Health , New Hyde Park, New York MD, PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Daniel King Sandeep Nadella 1 Northwell Health , New Hyde Park, New York 3 Cold Spring Harbor Laboratory, Cold Spring Harbor , New York MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: snadella{at}northwell.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Pancreatic cystic lesions (PCLs) are often discovered incidentally on imaging and may progress to pancreatic ductal adenocarcinoma (PDAC). PCLs have a high incidence in the general population, and adherence to screening guidelines can be variable. With the advent of technologies that enable automated text classification, we sought to evaluate various natural language processing (NLP) tools including large language models (LLMs) for identifying and classifying PCLs from radiology reports. We correlated our classification of PCLs to clinical features to identify risk factors for a positive PDAC biopsy. We contrasted a previously described NLP classifier to LLMs for prospective identification of PCLs in radiology. We evaluated various LLMs for PCL classification into low-risk or high-risk categories based on published guidelines. We compared prompt-based PCL classification to specific entity-guided PCL classification. To this end, we developed tools to deidentify radiology and track patients longitudinally based on their radiology reports. Additionally, we used our newly developed tools to evaluate a retrospective database of patients who underwent pancreas biopsy to determine associated factors including those in their radiology reports and clinical features using multivariable logistic regression modelling. Of 14,574 prospective radiology reports, 665 (4.6%) described a pancreatic cyst, including 175 (1.2%) high-risk lesions. Our Entity-Extraction Large Language Model tool achieved recall 0.992 (95% confidence interval [CI], 0.985-0.998), precision 0.988 (0.979-0.996), and F1-score 0.990 (0.985-0.995) for detecting cysts; F1-scores were 0.993 (0.987-0.998) for low-risk and 0.977 (0.952-0.995) for high-risk classification. Among 4,285 biopsy patients, 330 had pancreatic cysts documented ≥6 months before biopsy. In the final multivariable model (AUC = 0.877), independent predictors of adenocarcinoma were change in duct caliber with upstream atrophy (adjusted odds ratio [AOR], 4.94; 95% CI, 1.30-18.79), mural nodules (AOR, 11.02; 1.81-67.26), older age (AOR, 1.10; 1.05-1.16), lower body mass index (AOR, 0.86; 0.76-0.96), and total bilirubin (AOR, 1.81; 1.18-2.77). Automated NLP-based analysis of radiology reports using LLM-driven entity extraction can accurately identify and risk-stratify PCLs and, when retrospectively applied, reveal factors predicting malignant progression. Widespread implementation may improve surveillance and enable earlier intervention. Introduction Pancreatic cancer is among the leading causes of cancer-related death in the United States, with an estimated 66,440 new cases and 51,570 deaths in 2024. 1 – 3 Pancreatic ductal adenocarcinoma (PDAC) accounts for approximately 90% of cases. 4 Five-year survival remains dismal at about 10%, largely because most patients present at a late stage. 5 PDAC can arise from precursor lesions, including pancreatic cystic lesions (PCLs), which are often discovered incidentally on imaging and present an opportunity for earlier intervention. PCLs have an estimated global prevalence between 2-18%, increasing with age. 6 , 7 The majority are benign, yet certain mucinous subtypes, including intraductal papillary mucinous neoplasms (IPMNs) and mucinous cystic neoplasms (MCNs), carry significant malignant potential. 8 , 9 The American Gastroenterological Association (AGA) and American College of Gastroenterology (ACG) guidelines recommend distinguishing low-risk cysts suitable for surveillance from high-risk cysts that may require endoscopic evaluation or resection. 10 , 11 Despite these guidelines, real-world adherence varies, with many patients lacking appropriate surveillance or specialty referral. 12 – 14 Recent advances in artificial intelligence (AI), particularly natural language processing (NLP), allow for automated interpretation of radiology reports to detect PCLs. 15 Early rule-based NLP methods achieved high sensitivity, while modern deep learning, including transformer-based models, provide enhanced contextual understanding. 16 , 17 Large language models (LLMs) have demonstrated expert-level interpretation in certain medical text tasks, although their performance in evaluating pancreatic imaging findings remains unclear. 18 , 19 In this two-part study, we first evaluate and validate several NLP-based tools for identifying and classifying PCLs using existing computed tomography (CT) and magnetic resonance imaging (MRI) reports. We then apply these tools retrospectively to assess patients with biopsy-confirmed PDAC or nonmalignant findings, examining whether certain imaging features or clinical factors predict malignant transformation. We sought to demonstrate the potential of AI-driven radiology report analysis to improve surveillance and facilitate earlier intervention for PCLs. Methods Radiology Report Analysis and NLP Approaches Data were collected under Northwell Health Institutional Review Board guidelines. We queried all CT and MRI reports from 15 hospitals in our health system between February 14 and March 6, 2024, for patients 18 years or older, yielding 14,574 free-text reports. Each report was manually reviewed to identify pancreatic abnormalities and PCLs. A cyst was considered high-risk if, when present, the recommended next step in management was endoscopic ultrasound (EUS) ± fine-needle aspiration (FNA) per ACG recommendations. High-risk cysts met at least one of the following criteria: main pancreatic duct diameter or dilation >5 mm; cyst size ≥3 cm; cyst size ≥2 cm in lesions not described as IPMN or MCN; presence of a mural nodule or solid component; or a change in main duct caliber with upstream pancreatic atrophy. Several NLP techniques were applied to automate cyst identification and risk stratification. First, a previously described transformer-based model, iNav, was used to identify PCLs from radiology report text. 20 iNav was derived from RadBERT and fine-tuned to detect abnormal pancreatic imaging findings. 21 iNav assigns a single label to the report when an abnormality is present but does not perform risk stratification. We developed a PHI-compliant application that interfaces with the Northwell AI Hub API, which hosts multiple large-language models. The application securely sends each radiology report to the AI Hub, where the model parses the text according to predefined instructions. Two versions of the LLM tool were implemented. In Open-Prompt LLM (opLLM), the model received broad, free-text questions asking whether a cyst was present and, if so, whether it was high risk. We tested three prompt variants (Supplemental Table 1). In Entity-Extraction LLM (eeLLM), the model was prompted to determine whether a PCL was present and to extract specific features from the text, including cyst size and main duct diameter (Supplemental Figure 1). These extracted entities were then processed by an algorithm to risk-stratify each lesion. We compared iNav, opLLM Variant 1, and eeLLM for cyst identification, then tested all three opLLM variants against eeLLM for risk stratification. The prompts for opLLM and eeLLM were initially developed using the generative pretrained transformer 4 (GPT-4) LLM. We subsequently compared multiple LLMs (Gemini 1.0, Gemini 1.5 Flash, Gemini 1.5 Pro, PaLM 2, GPT-4o, and GPT-4) using the best-performing LLM tool. Retrospective Evaluation of PCLs and Subsequent PDAC Risk We retrospectively analyzed all patients 18 years or older who underwent pancreatic biopsy at the same 15 hospitals between January 2019 and November 2024. Biopsies confirming PDAC were deemed positive. For multiple biopsies, we used the earliest positive or most recent negative date. Patients without prior CT or MRI of the abdomen or chest, or whose first imaging occurred within six months of the biopsy, were excluded. For the remaining patients, all radiology reports predating the biopsy date (August 2016 to November 2024) were manually reviewed and parsed with eeLLM. Patients whose imaging showed only normal pancreatic findings or solid pancreatic masses, or whose cyst was first identified within six months of biopsy, were excluded to allow sufficient time for potential cyst progression. The final cohort consisted solely of patients with a PCL, classified as high-risk or low-risk according to the criteria described above. We separated biopsy-positive and biopsy-negative patients to compare demographics, imaging features, and laboratory values. Statistical Analysis Statistical analyses were conducted in Python version 3.11.5. Analyses with bilateral P < 0.05 were considered statistically significant. Performance was evaluated using recall, precision, and F1-score. The 95% confidence intervals (CIs) were computed with bootstrapping (10,000 samples) using SciPy 1.13. Demographics were evaluated using chi-square (sex, race) or Mann-Whitney tests (age, body mass index [BMI]). Univariable logistic regression was used to calculate odds ratios (ORs) with 95% CI for imaging features. Serum labs were analyzed via logistic regression and Mann-Whitney tests, using the time closest to cyst identification (up to one year before or one month after). Continuous laboratory variables with absolute skewness greater than 1 were log-transformed using the natural logarithm. These analyses were repeated in both high-risk and low-risk subgroups. Finally, three multivariable logistic regression models were created: (1) imaging features only, (2) imaging features and demographics, and (3) imaging features, demographics, and labs. Adjusted odds ratios (AOR) with 95% CI were calculated. Model discrimination was evaluated with area under the curve (AUC) via receiver operating characteristic (ROC) curves and calibration with the Hosmer-Lemeshow (H-L) statistic. Results Radiology Report Analysis and NLP Approaches Prevalence of Pancreatic Cysts in Prospective Radiology Reports A total of 14,574 prospectively obtained free-text radiology reports from 13,410 patients were manually reviewed, of which 2,301 (15.8%) contained a pancreatic abnormality (Supplemental Table 2). Among these, 665 (4.6%) reports described a pancreatic cyst, with 175 (1.2%) high-risk and 490 (3.4%) low-risk cysts. Overall, 612 (4.6%) of the 13,410 unique patients had a cyst, including 159 (1.2%) high-risk PCLs. eeLLM Demonstrates Superior Cyst Identification For cyst detection ( Figure 1a ), eeLLM achieved highest performance with recall 0.992 (95% CI, 0.985-0.998), precision 0.988 (0.979-0.996), and F1-score 0.990 (0.985-0.995). opLLM Variant 1 performed similarly with recall 0.991 (95% CI, 0.983-0.997), precision 0.972 (0.959-0.984), and F1-score 0.981 (0.974-0.988). iNav showed substantially lower performance with recall 0.857 (95% CI, 0.830-0.883), precision 0.551 (0.520-0.581), and F1-score 0.671 (0.644-0.695). Download figure Open in new tab Figure 1. Performance Metrics of Automated Tools for Pancreatic Cyst Identification from Free-Text Radiology Reports. Bar graphs display recall, precision, and F1-scores with 95% CIs for (A) three NLP-based tools identifying pancreatic cysts (n=665) from free-text radiology reports (n=14,574) and (B) six LLMs performing cyst identification via the eeLLM tool. Abbreviations: CI, confidence intervals; eeLLM, Entity-Extraction Large Language Model; NLP, natural language processing; opLLM, Open-Prompt Large Language Model. GPT-4 Outperforms Other LLMs in Cyst Identification When comparing the six LLMs using eeLLM for cyst identification ( Figure 1b ), GPT-4 was the strongest performing model with recall 0.992 (95% CI, 0.985-0.998), precision 0.988 (0.979-0.996), and F1-score 0.990 (0.985-0.995). GPT-4o and Gemini 1.5 Pro performed similarly, with recalls 0.988 (95% CI, 0.979-0.996) and 0.985 (0.975-0.994), whereas Gemini 1.0 and PaLM 2 showed lower recalls of 0.195 (0.165-0.226) and 0.191 (0.161-0.221). eeLLM Outperforms opLLM for Pancreas Cyst Classification Our eeLLM tool outperformed all opLLM variants in classifying low-risk and high-risk cysts ( Figure 2a and 2b ). For low-risk lesions, the eeLLM tool achieved highest recall 0.996 (95% CI, 0.989-1.000), precision 0.990 (0.980-0.998), and F1-score 0.993 (0.987-0.998); opLLM variants had recalls from 0.955 to 0.976 and lower F1-scores. A similar pattern appeared for high-risk classification, with eeLLM reaching recall 0.977 (95% CI, 0.952-0.995), precision 0.977 (0.953-0.995), and F1-score 0.977 (0.960-0.992), whereas opLLM variants ranged from recalls 0.171 to 0.389. Download figure Open in new tab Figure 2. Performance Metrics of Automated Tools for Pancreatic Cyst Risk Stratification from Free-Text Radiology Reports. Bar graphs display recall, precision, and F1-scores with 95% CIs for (a) low-risk cyst (n=490) and (b) high-risk cyst (n=175) stratification from free-text radiology reports (n=14,574) by the eeLLM tool and three opLLM variants, and for (c) low-risk cyst and (d) high-risk cyst stratification by six LLMs via the eeLLM tool. Abbreviations: CIs, confidence interval; eeLLM, Entity-Extraction Large Language Model; NLP, natural language processing; opLLM, Open-Prompt Large Language Model. GPT-4 Outperforms Other LLMs in Cyst Classification For low-risk classification ( Figure 2c ), GPT-4 again attained highest recall 0.996 (95% CI, 0.989-1.000), precision 0.990 (0.980-0.998), and F1-score 0.993 (0.987-0.998), followed by GPT-4o with recall 0.980 (0.966-0.992), precision 0.918 (0.893-0.940), and F1-score 0.948 (0.933-0.961). Gemini 1.0 and PaLM 2 had much lower recalls of 0.176 (95% CI, 0.143-0.209) and 0.204 (0.169-0.240), respectively. For high-risk cyst classification ( Figure 2d ), GPT-4 achieved highest recall 0.977 (95% CI, 0.952-0.995), precision 0.977 (0.953-0.995), and F1-score 0.977 (0.960-0.992), followed by GPT-4o with recall 0.943 (0.907-0.975), precision 0.932 (0.892-0.968), and F1-score 0.938 (0.909-0.962). Gemini 1.0 and PaLM 2 performed poorly with recalls 0.137 (95% CI, 0.087-0.190) and 0.080 (0.041-0.122), respectively. eeLLM Tool Identifies BMI and Age as Risk Factors for a Positive PDAC Biopsy in Patients with preexisting PCLs Cohort Characteristics Between January 2019 and November 2024, 4,285 patients in our health system underwent pancreatic biopsy. After excluding those without prior imaging or whose first imaging study occurred within six months of the most recent biopsy, 1,059 remained. Of these, 411 had pancreatic cysts (275 high-risk, 136 low-risk), whereas 648 were excluded for having only normal pancreatic features or non-cystic abnormalities, or a concurrent solid pancreatic mass. An additional 81 patients were excluded because their cyst was first documented within six months of biopsy, yielding a final cohort of 330. Among these 330, 225 were high-risk (25 PDAC-positive, 200 PDAC-negative), and 105 were low-risk (10 PDAC-positive, 95 PDAC-negative). Figure 3 presents a flow diagram of these steps. We assessed our eeLLM tool on the cohort of 1,059 patients, yielding similar performance to the first cohort with recall 0.990 (95% CI, 0.980-0.998), precision 0.991 (0.983-1.000), and F1-score 0.991 (0.984-0.997) for cyst identification, recall 0.956 (0.918-0.986), precision 0.971 (0.938-0.993), and F1-score 0.963 (0.938-0.984) for low-risk classification, and recall 0.978 (0.960-0.993), precision 0.979 (0.959-0.993), and F1-score 0.979 (0.965-0.989) for high-risk classification. Download figure Open in new tab Figure 3. Retrospective Study Flow Diagram. Abbreviations: CT, computed tomography; MR, magnetic resonance; PDAC, pancreatic ductal adenocarcinoma. Evaluation of risk factors associated with a positive biopsy in PCLs Table 1 indicates that sex did not differ by PDAC status in any group, and race was also nonsignificant (Supplemental Table 3 for race distribution). Age was consistently higher in PDAC-positive patients, with median ages of 78 vs 71 years in the overall cyst and low-risk cohorts ( P <0.001 and P =0.044), and 77 vs 70 years ( P <0.001) in the high-risk subgroup. Median BMI was significantly lower in PDAC-positive patients in the entire cyst cohort (24.7 vs 28.2, P <0.001) and high-risk subgroup (24.7 vs 28.5, P <0.001) but not in the low-risk group. View this table: View inline View popup Download powerpoint Table 1. Demographic characteristics for PDAC-positive versus PDAC-negative patients. Sex and Race were evaluated using chi-square tests; Age and BMI were evaluated using the Mann-Whitney U test. Significant differences ( P <0.05) are marked with an asterisk (*). Abbreviations : BMI, body mass index; IQR, interquartile range; PDAC, pancreatic ductal adenocarcinoma. Supplemental Table 4 shows results of univariable logistic regression of seven imaging features (Supplemental Table 5 for exact counts). Because cyst size, main duct dilation, duct caliber change with atrophy, and mural nodules define a high-risk cyst, these variables were not separately analyzed in low-risk lesions. Duct dilation was observed in 45.7% of PDAC-positive vs 18.6% of PDAC-negative patients (OR 3.67; 95% CI 1.78-7.60; P =0.001) overall, and 64.0% vs 27.5% (4.69; 1.96-11.23; P <0.001) in high-risk cysts. Duct caliber change with atrophy appeared in 25.7% vs 8.1% (OR 3.91; 95% CI 1.65-9.29; P =0.004) overall, and 36.0% vs 12.0% (4.12; 1.64-10.36; P =0.004) in high-risk cysts. Mural nodules were found in 14.3% vs 3.4% (OR 4.75; 95% CI 1.52-14.82; P =0.014) overall, rising to 20.0% vs 5.0% (4.75; 1.48-15.28; P =0.016) in high-risk cysts. Although cyst size was nonsignificant (OR 0.51; 95% CI 0.25-1.03; P =0.070) overall, it reached significance (0.20; 0.08-0.51; P =0.001) in high-risk PCLs. Enhancing cyst wall, septations, and pancreatitis were not significant. Among log-transformed laboratory values (Supplemental Table 6), only carbohydrate antigen (CA) 19-9 was significantly associated with a diagnosis of PDAC in the entire cohort (OR 3.42; 95% CI 1.24-9.45; P =0.018) and high-risk subgroup (OR 3.20; 1.18-8.67; P =0.023). Mann-Whitney comparisons (Supplemental Table 7) of CA 19-9 values were also higher in PDAC-positive patients (70.85 vs 12.0 U/mL; P =0.006 overall; 70.85 vs 12.0 U/mL; P =0.009 high-risk subgroup). In the low-risk subgroup, CA 19-9, carcinoembryonic antigen (CEA), and alpha-fetoprotein (AFP) lacked sufficient observations for analysis. No other laboratory parameters were significant. Laboratory study counts are provided in Supplemental Table 7. Multi-modal integration of pathology, radiology reports and laboratory information reveals BMI and age as associated risk factors for a positive biopsy in PCLs In Model A (n=330, imaging features only), main duct diameter (AOR 2.65; 95% CI 1.13-6.19; P =0.024), mural nodules (4.74; 1.40-16.08; P =0.013), and cyst size (0.41; 0.19-0.88; P =0.021) were significant, while duct caliber change trended toward significance. Model B (n=330, imaging and demographics) confirmed age (AOR 1.08; 95% CI 1.04-1.12; P <0.001), BMI (0.91; 0.83-0.98; P =0.017), main duct diameter (2.60; 1.04-6.53; P =0.042), mural nodules (5.21; 1.29-20.98; P =0.020), and cyst size (0.42; 0.19-0.94; P =0.036) as independent predictors, while duct caliber change was not significant. In Model C (n=261, imaging, demographics, and labs), CA 19-9 was excluded due to near-complete separation. Total bilirubin (AOR 1.81; 95% CI 1.18-2.77; P =0.006) became significant, while main duct diameter (1.52; 0.44-5.30; P =0.512) and cyst size (0.36; 0.13-1.01; P =0.052) lost significance. Change in duct caliber with atrophy emerged as an independent factor (AOR 4.94; 95% CI 1.30-18.79; P =0.019), along with mural nodules (11.02; 1.81-67.26; P =0.009), age (1.10; 1.05-1.16; P <0.001), and BMI (0.86; 0.76-0.96; P =0.008). We observed a moderate correlation between main duct diameter and duct caliber change with atrophy (r=0.38), which may explain why main duct diameter lost significance when additional covariates were introduced. Figure 4 shows a forest plot with AOR and 95% CI of each model. ROC curves in Supplemental Figure 2 demonstrate Model C had the highest AUC of 0.877 versus the lowest AUC of 0.721 for Model A, while calibration plots in Supplemental Figure 3 demonstrated acceptable calibration across all models with HL P ≥0.665. Download figure Open in new tab Figure 4. Forest plot containing adjusted OR with 95% CI for three multivariable logistic regression models. Abbreviations: BMI, body mass index. Discussion We leveraged NLP-based tools to identify and risk-stratify PCLs from free-text radiology reports, then applied these tools to assess factors predicting malignant progression. Two prior studies investigated using automated NLP to detect PCLs: one employed rule-based expressions while another deployed a transformer-based model. 16 , 22 Our study expands on this by being among the first to investigate using LLMs to perform cyst identification and risk stratification, evaluate how variations in prompting strategies can influence LLM performance of these tasks, and integrate both tasks into a single pipeline. In our initial analysis, both the eeLLM and opLLM tools outperformed the transformer-based iNav for cyst identification, likely reflecting the superior language understanding of LLMs. eeLLM consistently achieved performance near manual review for cyst detection and risk classification, whereas the best opLLM variant performed significantly worse in classification. These results suggest that prompting LLMs to extract discrete features from text is superior to broader, open-ended instructions. When tested across multiple LLMs, GPT-4 and GPT-4o yielded higher recall and F1-scores, likely reflecting their larger parameter sets. Notably, GPT-4 outperformed GPT-4o, suggesting that newer versions do not always guarantee superior performance if prompts are not tailored to them, and as such prompt design should be reevaluated whenever adopting a new LLM. For our second analysis, an initial cohort of 4,285 patients who underwent pancreatic biopsy was narrowed to 330 individuals who had pancreatic cysts identified on imaging at least six months before biopsy. Of these 330, 25 of 225 high-risk cysts (11.1%) and 10 of 105 low-risk cysts (9.5%) were positive for PDAC, highlighting the need to better understand the reasons for which low-risk lesions progress. Univariable analysis showed that older age and lower BMI were associated with PDAC across the overall cohort, findings that persisted in the final multivariable models. Age correlates with increasing PCL prevalence and malignant transformation, with studies reporting higher mean age in malignancy (67 vs 62 years, P <0.05) and a hazard ratio of 1.03 per year (95% CI, 1.00-1.06). 23 , 24 Conversely, higher BMI has been correlated with increased malignancy risk, though it should be noted that these studies specifically assessed patients with IPMNs or MCNs. 25 , 26 CA 19-9 was the only laboratory parameter associated with PDAC in univariable analysis, with significantly higher values in PDAC-positive patients. These results align with two meta-analyses that demonstrated CA 19-9 as a predictor of malignancy in pancreatic cystic neoplasms. 27 , 28 Although CA 19-9 was excluded from our final multivariable model due to near-complete separation, total bilirubin remained an independent predictor. Other investigations have found total bilirubin >1.2 mg/dL significant in univariable analysis of IPMNs, implying that bilirubin may also guide risk assessment. 29 Main duct diameter >5 mm, duct caliber change with atrophy, and mural nodules showed strong PDAC associations, aligning with established high-risk criteria. 10 , 11 After adjusting for demographics and labs, main duct diameter lost significance, whereas change in duct caliber with atrophy emerged as an independent predictor. Mural nodules remained significant across all models. Cyst size showed an inverse association with PDAC in multivariable models, in contrast to evidence that larger cysts have greater malignant potential. This discrepancy may result from selection bias in a retrospective design, as larger cysts are more likely to be biopsied, leaving smaller lesions underrepresented in our study. There are other limitations to this study. Although our NLP tools performed well within our health system, external validation is needed to assess generalizability. Moreover, performance ultimately depends on the radiologist; if an imaging feature is not documented, the LLM cannot identify it. The second phase yielded 330 patients, with only 10 low-risk cysts progressing to malignancy, restricting power in that subgroup. A prospective study is underway within our health system to actively identify PCLs, facilitate specialist referral, and encourage earlier interventions. LLM-based entity extraction achieves outcomes comparable to manual review for PCL identification and risk stratification. Retrospective evaluation revealed factors predicting malignant progression, including older age, lower BMI, duct caliber change with upstream atrophy, mural nodules, and total bilirubin. AI-driven tools could enhance surveillance and expedite interventions for high-risk PCLs. Further multi-institutional investigations are needed to confirm generalizability. Data Availability All data produced in the present study are available upon reasonable request to the authors. Conflict of Interest Disclosures None reported. Funding/Support The study did not receive any external funding. S.N. was supported by the Donaldson Fellowship. Acknowledgements We thank the Innovation team (Northwell Health, New Hyde Park, New York) for developing and maintaining the Northwell AI Hub platform and its application programming interface. None of these individuals received financial compensation for their contributions. References 1. ↵ Siegel RL , Miller KD , Wagle NS , Jemal A. Cancer statistics, 2023 . CA Cancer J Clin . 2023 ; 73 ( 1 ): 17 – 48 . doi: 10.3322/caac.21763 OpenUrl CrossRef PubMed 2. National Cancer Institute . SEER Cancer Stat Facts [Internet] . Bethesda (MD ): National Cancer Institute ; 2024 [cited 2025 June 11]. Available from: https://seer.cancer.gov/statfacts/ . 3. ↵ Rahib L , Smith BD , Aizenberg R , Rosenzweig AB , Fleshman JM , Matrisian LM . Projecting cancer incidence and deaths to 2030: the unexpected burden of thyroid, liver, and pancreas cancers in the United States [published correction appears in Cancer Res. 2014 Jul 15;74(14):4006] . Cancer Res . 2014 ; 74 ( 11 ): 2913 – 2921 . doi: 10.1158/0008-5472.CAN-14-0155 OpenUrl Abstract / FREE Full Text 4. ↵ Stoffel EM , Brand RE , Goggins M. Pancreatic Cancer: Changing Epidemiology and New Approaches to Risk Assessment, Early Detection, and Prevention . Gastroenterology . 2023 ; 164 ( 5 ): 752 – 765 . doi: 10.1053/j.gastro.2023.02.012 OpenUrl CrossRef PubMed 5. ↵ Mizrahi JD , Surana R , Valle JW , Shroff RT . Pancreatic cancer . Lancet . 2020 ; 395 ( 10242 ): 2008 – 2020 . doi: 10.1016/S0140-6736(20)30974-0 OpenUrl CrossRef PubMed 6. ↵ Gonda TA , Cahen DL , Farrell JJ . Pancreatic Cysts . N Engl J Med . 2024 ; 391 ( 9 ): 832 – 843 . doi: 10.1056/NEJMra2309041 OpenUrl CrossRef PubMed 7. ↵ Vilela A , Quingalahua E , Vargas A , et al. Global Prevalence of Pancreatic Cystic Lesions in the General Population on Magnetic Resonance Imaging: A Systematic Review and Meta-analysis . Clin Gastroenterol Hepatol . 2024 ; 22 ( 9 ): 1798 - 1809.e6 . doi: 10.1016/j.cgh.2024.02.018 OpenUrl CrossRef PubMed 8. ↵ Hasan A , Visrodia K , Farrell JJ , Gonda TA . Overview and comparison of guidelines for management of pancreatic cystic neoplasms . World J Gastroenterol . 2019 ; 25 ( 31 ): 4405 – 4413 . doi: 10.3748/wjg.v25.i31.4405 OpenUrl CrossRef PubMed 9. ↵ Scholten L , van Huijgevoort NCM , van Hooft JE , Besselink MG , Del Chiaro M. Pancreatic Cystic Neoplasms: Different Types, Different Management, New Guidelines . Visc Med . 2018 ; 34 ( 3 ): 173 – 177 . doi: 10.1159/000489641 OpenUrl CrossRef PubMed 10. ↵ Lennon AM , Ahuja N , Wolfgang CL . AGA Guidelines for the Management of Pancreatic Cysts . Gastroenterology . 2015 ; 149 ( 3 ): 825 . doi: 10.1053/j.gastro.2015.05.062 OpenUrl CrossRef PubMed 11. ↵ Elta GH , Enestvedt BK , Sauer BG , Lennon AM . ACG Clinical Guideline: Diagnosis and Management of Pancreatic Cysts . Am J Gastroenterol . 2018 ; 113 ( 4 ): 464 – 479 . doi: 10.1038/ajg.2018.14 OpenUrl CrossRef PubMed 12. ↵ Schubach A , Kothari S , Kothari T. Pancreatic Cystic Neoplasms: Diagnosis and Management . Diagnostics (Basel) . 2023 ; 13 ( 2 ): 207 . Published 2023 Jan 5. doi: 10.3390/diagnostics13020207 OpenUrl CrossRef PubMed 13. Westerveld D , Goddard A , Harris N , et al. Survey Study on the Practice Patterns of the Evaluation and Management of Incidental Pancreatic Cysts . Dig Dis Sci . 2019 ; 64 ( 3 ): 689 – 697 . doi: 10.1007/s10620-018-5368-x OpenUrl CrossRef PubMed 14. ↵ Schenck RJ , Miller FH , Keswani RN . The Surveillance Patterns of Incidentally Detected Pancreatic Cysts Vary Widely and Infrequently Adhere to Guidelines . Pancreas . 2019 ; 48 ( 7 ): 883 – 887 . doi: 10.1097/MPA.0000000000001352 OpenUrl CrossRef PubMed 15. ↵ Locke S , Bashall A , Al-Adely S , et al. Natural language processing in medicine: a review . Trends Anaesth Crit Care . 2021 ; 38 : 4 – 9 . doi: 10.1016/j.tacc.2021.02.007 OpenUrl CrossRef 16. ↵ Roch AM , Mehrabi S , Krishnan A , et al. Automated pancreatic cyst screening using natural language processing: a new tool in the early detection of pancreatic cancer . HPB (Oxford) . 2015 ; 17 ( 5 ): 447 – 453 . doi: 10.1111/hpb.12375 OpenUrl CrossRef PubMed 17. ↵ Al-Haddad MA , Friedlin J , Kesterson J , Waters JA , Aguilar-Saavedra JR , Schmidt CM . Natural language processing for the development of a clinical registry: a validation study in intraductal papillary mucinous neoplasms . HPB (Oxford) . 2010 ; 12 ( 10 ): 688 – 695 . doi: 10.1111/j.1477-2574.2010.00235.x OpenUrl CrossRef PubMed 18. ↵ Bhayana R , Nanda B , Dehkharghanian T , et al. Large Language Models for Automated Synoptic Reports and Resectability Categorization in Pancreatic Cancer . Radiology . 2024 ; 311 ( 3 ): e233117 . doi: 10.1148/radiol.233117 OpenUrl CrossRef PubMed 19. ↵ Bhayana R. Chatbots and Large Language Models in Radiology: A Practical Primer for Clinical and Research Applications . Radiology . 2024 ; 310 ( 1 ): e232756 . doi: 10.1148/radiol.232756 OpenUrl CrossRef PubMed 20. ↵ John KM , Tenner J , Franco Y , et al. Implementation of a real-time AI-guided navigation service for pancreas cancer . Cancer Res . 2024 ; 84 ( 2 suppl ): PR01 . doi: 10.1158/1538-7445.PANCA2023-PR01 OpenUrl CrossRef 21. ↵ Yan A , McAuley J , Lu X , et al. RadBERT: Adapting Transformer-based Language Models to Radiology . Radiol Artif Intell . 2022 ; 4 ( 4 ): e210258. Published 2022 Jun 15. doi: 10.1148/ryai.210258 OpenUrl CrossRef PubMed 22. ↵ Yamashita R , Bird K , Cheung PY , et al. Automated Identification and Measurement Extraction of Pancreatic Cystic Lesions from Free-Text Radiology Reports Using Natural Language Processing . Radiol Artif Intell . 2021 ; 4 ( 2 ): e210092. Published 2021 Dec 22. doi: 10.1148/ryai.210092 OpenUrl CrossRef PubMed 23. ↵ Lee CJ , Scheiman J , Anderson MA , et al. Risk of malignancy in resected cystic tumors of the pancreas < or =3 cm in size: is it safe to observe asymptomatic patients? A multi-institutional report . J Gastrointest Surg . 2008 ; 12 ( 2 ): 234 – 242 . doi: 10.1007/s11605-007-0381-y OpenUrl CrossRef PubMed 24. ↵ Youssef FF , Liu L , Lin W , et al. Pancreatic cyst features predict future development of pancreatic cancer: results of a nested case-control study . Gastrointest Endosc . 2024 ; 99 ( 2 ): 262 .e1-262.e9. doi: 10.1016/j.gie.2023.10.038 OpenUrl CrossRef 25. ↵ Sturm EC , Roch AM , Shaffer KM , et al. Obesity increases malignant risk in patients with branch-duct intraductal papillary mucinous neoplasm . Surgery . 2013 ; 154 ( 4 ): 803 – 809 . doi: 10.1016/j.surg.2013.07.011 OpenUrl CrossRef PubMed 26. ↵ Capurso G , Crippa S , Vanella G , et al. Factors Associated With the Risk of Progression of Low-Risk Branch-Duct Intraductal Papillary Mucinous Neoplasms . JAMA Netw Open . 2020 ; 3 ( 11 ): e2022933 . Published 2020 Nov 2. doi: 10.1001/jamanetworkopen.2020.22933 OpenUrl CrossRef 27. ↵ Cao S , Hu Y , Gao X , Liao Q , Zhao Y. Serum Carbohydrate Antigen 19-9 in Differential Diagnosis of Benign and Malignant Pancreatic Cystic Neoplasms: A Meta-Analysis . PLoS One . 2016 ; 11 ( 11 ): e0166406 . Published 2016 Nov 11. doi: 10.1371/journal.pone.0166406 OpenUrl CrossRef PubMed 28. ↵ Wang W , Zhang L , Chen L , et al. Serum carcinoembryonic antigen and carbohydrate antigen 19-9 for prediction of malignancy and invasiveness in intraductal papillary mucinous neoplasms of the pancreas: A meta-analysis . Biomed Rep . 2015 ; 3 ( 1 ): 43 – 50 . doi: 10.3892/br.2014.376 OpenUrl CrossRef PubMed 29. ↵ Shin SH , Han DJ , Park KT , Kim YH , Park JB , Kim SC . Validating a simple scoring system to predict malignancy and invasiveness of intraductal papillary mucinous neoplasms of the pancreas . World J Surg . 2010 ; 34 ( 4 ): 776 – 783 . doi: 10.1007/s00268-010-0416-5 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted July 17, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study Anthony Papale , Robert Flattau , Nandan Vithlani , Deepti Mahajan , Yonah Ziemba , Tiffany Zavadsky , Anthony Carvino , Daniel King , Sandeep Nadella medRxiv 2025.07.15.25331413; doi: https://doi.org/10.1101/2025.07.15.25331413 Share This Article: Copy Citation Tools Large Language Model-Based Entity Extraction Reliably Classifies Pancreatic Cysts and Reveals Predictors of Malignancy: A Cross-Sectional and Retrospective Cohort Study Anthony Papale , Robert Flattau , Nandan Vithlani , Deepti Mahajan , Yonah Ziemba , Tiffany Zavadsky , Anthony Carvino , Daniel King , Sandeep Nadella medRxiv 2025.07.15.25331413; doi: https://doi.org/10.1101/2025.07.15.25331413 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Gastroenterology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15228) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6599) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9231) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a00715352d74f67c',t:'MTc3OTU3MDk5OA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();