Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator

Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator Nathan Wolfrath , Gopika SenthilKumar , Adhitya Ramamurthi , View ORCID Profile Anai N. Kothari doi: https://doi.org/10.1101/2025.04.18.25325614 Nathan Wolfrath 1 Department of Surgery, Division of Surgical Oncology, Medical College of Wisconsin Find this author on Google Scholar Find this author on PubMed Search for this author on this site Gopika SenthilKumar 1 Department of Surgery, Division of Surgical Oncology, Medical College of Wisconsin Find this author on Google Scholar Find this author on PubMed Search for this author on this site Adhitya Ramamurthi 1 Department of Surgery, Division of Surgical Oncology, Medical College of Wisconsin Find this author on Google Scholar Find this author on PubMed Search for this author on this site Anai N. Kothari 1 Department of Surgery, Division of Surgical Oncology, Medical College of Wisconsin 2 Data Science Institute, Medical College of Wisconsin 3 Bud and Sue Selig Hub for Surgical Data Science, Medical College of Wisconsin Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Anai N. Kothari For correspondence: akothari{at}mcw.edu Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Large language models (LLMs) have shown capabilities in generating functional code, yet their utility in the development of clinical prediction tools has not been significantly explored. We evaluated GPT-4o’s capability to create a postoperative complication risk calculator similar to the existing National Surgical Quality Improvement Program (NSQIP) risk calculator. This included data preprocessing, predictive modeling, and development of a web application. Synthetic data of a similar structure to the NSQIP dataset was used when communicating with GPT-4o to maintain privacy. 512 lines of Python code were generated across 14 prompts, with one line requiring human editing. The resulting logistic regression models achieved similar Brier scores compared to the original NSQIP risk calculator and demonstrated strong discrimination (C-statistic > 0.75), while slightly underperforming previously reported predictive metrics for some outcomes. Development was completed in three hours. These findings suggest that LLMs can facilitate rapid development of clinical decision support tools, though output still requires human oversight and refinement. Introduction Large language models (LLMs) are an effective tool for generating operational computer code in response to a user prompt [ 1 , 2 ]. To date, the utility of these tools in the development of predictive clinical tools has not been sufficiently evaluated. In this study, we aim to understand the capability of LLMs in this capacity by using an LLM to generate comprehensive code for a postoperative complication risk calculator. This includes data preprocessing, model training, creation of a user-friendly, web-based application, and comparison of performance against a commonly-used risk calculator. Methods OpenAI’s GPT-4o was used for generation of Python3 code. The analytic approach and application were designed based on the American College of Surgeons (ACS) National Surgical Quality Improvement Program (NSQIP) Universal Surgical Risk Calculator [ 3 ]. An identical dataset to that used in the original calculator was used as source data, compiled from the NSQIP Participant Use File (PUF) 2009-2012. To preserve data privacy, a synthetic dataset (NSQIP-PUF-TWIN) was created leveraging Copulas [ 4 ] for multivariate modeling and sampling of continuous variables, and custom code to simulate categorical variables. All data sent online to GPT-4 was synthetic and included no real patient information, with AI-generated code applied to the real dataset locally. Development included three tasks: Data Preprocessing Encoding of categorical variables, generating columns such as patient BMI and Current Procedural Terminology (CPT) specific risk scores for each complication, and saving the transformed data. Model Creation and Evaluation Training of logistic regression models to predict target complications. Data was split into training (80%) and testing (20%) sets. Input features and performance metrics (C-statistic and Brier Score) were based on those described in the development of the original NSQIP risk calculator. Confidence intervals were estimated using 1000 sample bootstrapping. The models were compared against previously published NSQIP risk calculator performance (prior to recent conversion of the calculator to a machine learning approach). Web Application Development Creation of a web-based interface for entry of patient parameters and viewing calculated probability of each complication. Results 1,414,006 patients were included, with 1,131,204 records used for model training. For data preprocessing, 104 lines of code were generated by GPT-4o using four prompts. Four prompts were used to generate 93 lines of code for predictive modeling, and six prompts were used to generate 315 lines of code for the web-based application. Upon review, code was nearly universally correct, with one line (0.19%) requiring human editing. A full transcript of the LLM interactions and generated code are available at https://github.com/AnaiLab/LLMCalculator . Discrimination of the LLM-generated risk calculator (GPT-NSQIP) was high across all measured outcomes (C-statistic > 0.75). When compared to the published performance of the Universal NSQIP Surgical Risk Calculator, GPT-NSQIP achieved equivalent or superior performance by Brier score across all outcomes except surgical site infection, and similar but inferior performance by C-Statistic for all outcomes except post-operative pneumonia ( Table 1 ). View this table: View inline View popup Download powerpoint Table 1a: Target variables and model performance metrics for GPT-NSQIP compared to Bilimoria et al. ACS NSQIP calculator. Ranges represent a 95% confidence interval (not reported in original publication) View this table: View inline View popup Download powerpoint Table 1b: Input variables used for GPT-NSQIP models. Additionally, a functional web-based application was created ( Figure 1a/1b ). Total development time was 3 hours. Download figure Open in new tab Figure 1a: Portion of the input page of the generated application Download figure Open in new tab Figure 1b: Example results page of the GPT-4 generated application. Discussion Our results indicate that LLMs can assist in the creation of surgical risk estimation tools that achieve similar performance to existing clinically utilized models. All metrics indicate strong predictive performance, and in some cases are superior (i.e. lower Brier score) to previously published results. In areas where underperforming, GPT-NSQIP results still displayed similar results and impressive discrimination. Of note, estimated confidence intervals were not available from the comparison study, which somewhat limits interpretation of metrics. Additional possible reasons for non-identical metrics include differences in training and test dataset splitting, differences in methodology for incorporating procedure-specific risk, and full reliance on LLM- based code with minimal human modification. While the final model and generated application necessitate refinement and human review prior to clinical adoption, the ease and pace of development provides an opportunity for rapid experimentation and development in a variety of healthcare modeling applications. Data Availability Data in this study is from the National Surgical Quality Improvement Program (NSQIP) database and can be requested by investigators at participating institutions at facs.org. Refesrences 1. ↵ Sakib FA , Khan SH , Karim AHMR. Extending the Frontier of ChatGPT: Code Generation and Debugging . Published online July 17, 2023. doi: 10.48550/arXiv.2307.08260 OpenUrl CrossRef 2. ↵ Li Y , Choi D , Chung J , et al. Competition-level code generation with AlphaCode . Science . 2022 ; 378 ( 6624 ): 1092 – 1097 . doi: 10.1126/science.abq1158 OpenUrl CrossRef PubMed 3. ↵ Bilimoria KY , Liu Y , Paruch JL , et al. Development and Evaluation of the Universal ACS NSQIP Surgical Risk Calculator: A Decision Aid and Informed Consent Tool for Patients and Surgeons . Journal of the American College of Surgeons . 2013 ; 217 ( 5 ): 833 – 842 e3. doi: 10.1016/j.jamcollsurg.2013.07.385 OpenUrl CrossRef PubMed 4. ↵ Copulas . https://github.com/sdv-dev/Copulas View the discussion thread. Back to top Previous Next Posted April 22, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator Nathan Wolfrath , Gopika SenthilKumar , Adhitya Ramamurthi , Anai N. Kothari medRxiv 2025.04.18.25325614; doi: https://doi.org/10.1101/2025.04.18.25325614 Share This Article: Copy Citation Tools Development and Evaluation of an AI-Assisted, Privacy-Preserving Surgical Risk Calculator Nathan Wolfrath , Gopika SenthilKumar , Adhitya Ramamurthi , Anai N. Kothari medRxiv 2025.04.18.25325614; doi: https://doi.org/10.1101/2025.04.18.25325614 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (570) Allergy and Immunology (864) Anesthesia (301) Cardiovascular Medicine (4445) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1515) Epidemiology (15234) Forensic Medicine (30) Gastroenterology (1127) Genetic and Genomic Medicine (6610) Geriatric Medicine (669) Health Economics (999) Health Informatics (4545) Health Policy (1370) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1266) Infectious Diseases (except HIV/AIDS) (15925) Intensive Care and Critical Care Medicine (1104) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6612) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1147) Occupational and Environmental Health (957) Oncology (3340) Ophthalmology (975) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1694) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5458) Public and Global Health (9243) Radiology and Imaging (2204) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1197) Rheumatology (596) Sexual and Reproductive Health (715) Sports Medicine (530) Surgery (713) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a021440e59f2593a',t:'MTc3OTg0NTU0Nw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();