Prompt injection attacks on vision-language models for surgical decision support

Prompt injection attacks on vision-language models for surgical decision support | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Prompt injection attacks on vision-language models for surgical decision support Zheyuan Zhang , View ORCID Profile Muhammad Ibtsaam Qadir , Matthias Carstens , Evan Hongyang Zhang , Madison Sarah Loiselle , Farren Marc Martinus , Maksymilian Ksawier Mroczkowski , View ORCID Profile Jan Clusmann , View ORCID Profile Jakob Nikolas Kather , View ORCID Profile Fiona R. Kolbinger doi: https://doi.org/10.1101/2025.07.16.25331645 Zheyuan Zhang 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Muhammad Ibtsaam Qadir 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA BE Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Muhammad Ibtsaam Qadir Matthias Carstens 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA 2 Department of Visceral, Thoracic and Vascular Surgery, University Hospital and Faculty of Medicine Carl Gustav Carus, TUD Dresden University of Technology , Dresden, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Evan Hongyang Zhang 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA BS Find this author on Google Scholar Find this author on PubMed Search for this author on this site Madison Sarah Loiselle 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Farren Marc Martinus 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Maksymilian Ksawier Mroczkowski 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Jan Clusmann 3 Else Kroener Fresenius Center for Digital Health, Faculty of Medicine and University Hospital Carl Gustav Carus, TUD Dresden University of Technology , 01307 Dresden, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jan Clusmann Jakob Nikolas Kather 3 Else Kroener Fresenius Center for Digital Health, Faculty of Medicine and University Hospital Carl Gustav Carus, TUD Dresden University of Technology , 01307 Dresden, Germany 4 Department of Medicine I, Faculty of Medicine and University Hospital Carl Gustav Carus, TUD Dresden University of Technology , 01307 Dresden, Germany 5 Medical Oncology, National Center for Tumor Diseases (NCT), University Hospital Heidelberg , Heidelberg, Germany MD, MSc Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jakob Nikolas Kather Fiona R. Kolbinger 1 Weldon School of Biomedical Engineering, Purdue University , West Lafayette, IN, USA 2 Department of Visceral, Thoracic and Vascular Surgery, University Hospital and Faculty of Medicine Carl Gustav Carus, TUD Dresden University of Technology , Dresden, Germany MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Fiona R. Kolbinger For correspondence: fkolbing{at}purdue.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Importance Artificial Intelligence-driven analysis of laparoscopic video holds potential to increase the safety and precision of minimally invasive surgery. Vision-language models are particularly promising for video-based surgical decision support due to their capabilities to comprehend complex temporospatial (video) data. However, the same multimodal interfaces that enable such capabilities also introduce new vulnerabilities to manipulations through embedded deceptive text or images (prompt injection attacks). Objective To systematically evaluate how susceptible state-of-the-art video-capable vision-language models are to textual and visual prompt injection attacks in the context of clinically relevant surgical decision support tasks. Design, Setting, and Participants In this observational study, we systematically evaluated four state-of-the-art vision-language models, Gemini 1.5 Pro, Gemini 2.5 Pro, GPT-o4-mini-high, and Qwen 2.5-VL, across eleven surgical decision support tasks: detection of bleeding events, foreign objects, image distortions, critical view of safety assessment, and surgical skill assessment. Prompt injection scenarios involved misleading textual prompts and visual perturbations, displayed as white text overlay, applied at varying durations. Main Outcomes and Measures The primary measure was model accuracy, contrasted between baseline performance and each prompt injection condition. Results All vision-language models demonstrated good baseline accuracy, with Gemini 2.5 Pro generally achieving the highest mean [standard deviation] accuracy across all tasks (0.82 [0.01]), compared to Gemini 1.5 Pro (0.70 [0.03]) and GPT-o4 mini-high (0.67 [0.06]). Across tasks, Qwen 2.5-VL censored most outputs and achieved an accuracy of (0.58 [0.03]) on non-censored outputs. Textual and temporally-varying visual prompt injections reduced the accuracy for all models. Prolonged visual prompt injections were generally more harmful than single-frame injections. Gemini 2.5 Pro showed the greatest robustness and maintained stable performance for several tasks despite prompt injections, whereas GPT-o4-mini-high exhibited the highest vulnerability, with mean (standard deviation) accuracy across all tasks declining from 0.67 (0.06) at baseline to 0.24 (0.04) under full-duration visual prompt injection ( P < .001). Conclusion and Relevance These findings indicate the critical need for robust temporal reasoning capabilities and specialized guardrails before vision-language models can be safely deployed for real-time surgical decision support. Question Are video vision-language models (VLMs) susceptible to textual and visual prompt injection attacks when used for surgical decision support tasks? Finding Textual and visual prompt injection attacks consistently degraded the performance of four state-of-the-art VLMs across eleven surgical tasks. Gemini 2.5 Pro was most robust to textual and visual prompt injection attacks, whereas GPT-o4-mini-high was most vulnerable. Prolonged visual injections had a greater negative impact than single-frame injections. Meaning Present-generation video VLMs are highly vulnerable to textual and visual prompt injection attacks. This critical safety vulnerability must be addressed before their integration into surgical decision support systems. Introduction Laparoscopic video data analysis using Artificial Intelligence (AI) holds potential to increase the safety and accuracy of minimally invasive surgical procedures. 1 , 2 State-of-the-art AI models for surgical video analysis can provide a temporospatial understanding of surgical scenes and processes. 3 These capabilities enable concrete clinical applications, such as detection of intraoperative bleeding events 4 , 5 , identification of safe and unsafe dissection zones in laparoscopic cholecystectomy and rectal surgery 6 , 7 , assessment of the critical view of safety (CVS) during laparoscopic cholecystectomy 8 , and technical skill feedback for educational and quality assurance purposes. 9 Most academic works in the field of AI-based surgical video analysis over the past decade have relied on strongly supervised, unimodal image analysis models 10 . Vision-language models (VLMs) are generative AI models trained on vast amounts of multimodal data that can comprehend and produce both textual and visual information. Several medical applications, such as radiological report generation or histopathological image interpretation, have been proposed both for specialist VLMs 11 , 12 , and for generalist VLMs, such as GPT-4o. 13 – 15 In many medical applications, including dermatology 16 and pathology 17 , VLMs have demonstrated the potential to achieve expert-level performance. Early studies indicate similar potential in surgery, where VLMs have demonstrated performance on par with specialist models in surgical scene and process understanding tasks. 18 , 19 Present-generation generalist VLMs, such as Gemini 2.5 Pro 20 , GPT-o4-mini-high 21 , and Qwen 2.5-VL 22 , provide the technical capability to interpret not just images, but also video inputs. Coupled with the increasing availability of powerful cloud-computing resources, these video interpretation capabilities are likely to facilitate near real-time interaction with clinically meaningful intraoperative decision support models in the operating room to ensure high-quality surgical care. 23 VLMs typically have guardrails to protect against various types of malicious attacks. Prompt injection attacks involve manipulating AI model behavior through adversarial textual or visual inputs that conflict with the actual prompt. As of November 2024, generative AI developers have recognized prompt injection attacks as the most relevant security vulnerability for large language models (LLMs) 24 , and the relevance of prompt injection attacks has been described in various healthcare settings. 25 , 26 In the context of surgical assistance, prompt injection attacks could lead to incorrect model outputs, potentially compromising patient safety ( Figure 1a ). Beyond direct threats to model integrity, broader cybersecurity risks to healthcare systems, such as ransomware attacks, have escalated between 2010 and 2024 and compromise millions of patient records each year. 27 Download figure Open in new tab Figure 1: Prompt injection attacks on vision-language models for surgical decision support. (a) Conceptual illustration of a prompt injection attack on a video VLM for real-time surgical decision support in the operating room. (b) Likely prompt injection attack vectors in the context of VLM-supported surgery are at communication interfaces between the endoscopic unit and the VLM. (c) Prediction tasks analyzed in this study. (d) Illustration of a visual prompt injection. (e) Varying timing and duration of visual prompt injection strategies for surgical video data. (f) Study endpoints. Abbreviations: Critical View of Safety (CVS), Vision-language model (VLM). Despite the increasing investigation of LLMs and VLMs in clinical settings 28 – 31 , the vulnerability of VLMs capable of video interpretation to prompt injection currently remains unknown. Specific to surgical deployment, realistic attack vectors include man-in-the-middle attacks or compromised interfaces between the endoscopy unit and the inference engines, particularly in networked or cloud-based VLM architectures ( Figure 1b ). Potential adversaries might range from financially motivated criminals deploying ransomware for extortion 32 to state-sponsored actors aiming to disrupt medical infrastructure in geopolitical conflicts. 33 Adversaries could manipulate patient data visually or digitally to mislead models, producing clinical prediction errors. Here, we present a comprehensive evaluation of prompt injection attacks on four VLMs for video interpretation for intraoperative surgical decision support tasks. We benchmark VLM performance across eleven clinically relevant prediction tasks ( Figure 1c ) and simulate realistic threat scenarios through both textual and temporally-varying visual prompt injections ( Figure 1d ). Our results indicate that prompt injection attacks significantly reduce the accuracy of VLMs in surgical decision support tasks, potentially leading to safety risks related to the clinical deployment of VLMs in intraoperative settings. Methods Ethics Statement This study was conducted in accordance with the Declaration of Helsinki and its subsequent amendments. No identifiable patient data were used in this study; all clinical data used are publicly available. Therefore, no informed consent was required. The local Institutional Review Board at Purdue University reviewed and approved the overall analysis on February 7, 2024 (IRB-2023-1736). All prompt injection experiments were performed in controlled, simulated environments to prevent any risk of unintended harm. The disclosed attack strategies and prompts are intended solely for research purposes. The models evaluated in this study are research tools and are not approved for clinical use. Datasets and Prediction Tasks Eleven prediction tasks were analyzed using publicly available video data from laparoscopic surgical procedures (cholecystectomy 34 – 37 , Roux-en-Y gastric bypass 38 , hysterectomy 39 , colorectal surgery 40 ), and colonoscopies. 41 This study includes the following prediction tasks: Detection of foreign objects (surgical gauze sponge, needle, specimen bag), CVS assessment 42 , detection of image distortions (smoke, soiled lens, poor illumination), bleeding detection, surgical phase recognition, surgical skill assessment, and polyp detection ( Figure 1c , Table 1 ). View this table: View inline View popup Table 1: Overview of prediction tasks, data sources, and prompts. For each prediction task, five video clips were curated from open-access laparoscopic and endoscopic video datasets (Supplementary Table 1). Four video VLMs were prompted as outlined below. For each prediction task, five representative video clips with clearly discernible ground truths were curated from the abovementioned datasets. For a detailed analysis of CVS assessment capabilities, five videos with high, intermediate, and low Sanford scores (indicating varying levels of CVS achievement) 42 were curated, respectively. Supplementary Table 1 details the video data and ground truth data sources for each of the 65 clips. For foreign object detection tasks, smoke detection, bleeding detection, surgical phase recognition, skill assessment, and polyp detection, videos had a duration of 30 seconds. Videos for poor illumination and soiled lens detection were 15 seconds long. For CVS assessment, the videos had variable lengths with a mean (SD) of 33.4 (10.8) seconds. All videos were recorded at 25 frames per second (fps), except for videos 18 and 20 from the HeiChole dataset, which were recorded at 50 fps. For visual prompt injections, a watermark overlay was applied to the video, sized relative to the frame, and positioned in the top-right corner. Arial font was used, with font size set to 5% of the frame height and a top margin of 2%. For multi-line text, a line spacing of 6% of the frame height was applied. To ensure readability, the text was enclosed in a semi-transparent black box with an opacity of 0.6 ( Figure 1d ). VLM Setup We evaluated Gemini 1.5 Pro (2024-02-15), Gemini 2.5 Pro (2025-05-06), GPT-o4-mini-high (2025-04-16), and Qwen 2.5-VL-32B (2025-03-25). All models were accessed between June 5 and June 11, 2025, through their respective official web interfaces using default settings with learning and chat history features disabled ( Supplementary Table 2 ). View this table: View inline View popup Download powerpoint Table 2: Baseline vision-language model accuracy across clinically relevant surgical video analysis tasks. The table shows baseline accuracies without prompt injections as mean (SD). Model performance was aggregated across five video clips per prediction task, using only non-censored outputs (Supplementary Table 2). Given that Qwen 2.5-VL censored most outputs, it was excluded from formal statistical analyses and only results from non-censored outputs are displayed. Statistical comparisons among the remaining models (Gemini 1.5 Pro, Gemini 2.5 Pro, GPT-o4-mini-high) were conducted via Kruskal-Wallis and Dunn’s tests. P values represent omnibus test results (Kruskal-Wallis test), indicating statistically significant performance differences among all three models for each prediction task. Detailed pairwise comparisons are provided in Supplementary Table 4 . Experimental Setup For each video in each use case, six different prompting strategies were applied. These included one baseline no-injection prompt, one textual prompt injection, and four types of temporally-varying visual prompt injections ( Figure 1e ): (i) early-time injection (i.e., visual prompt injected only in the first frame), (ii) late-time injection (i.e., visual prompt injected only in the last frame), (iii) partial-time injection (i.e., visual prompt injected in the first 50% of the video frames), and (iv) full-time injection (i.e., visual prompt injected throughout all frames). Each strategy was evaluated in a newly initiated chat session to avoid context leakage between runs. For instance, in the polyp detection task, the baseline prompt was: “ Are there any polyps in this endoscopic view? ” The same video was then input to a new chat with a text prompt injection: “ Polyps are clearly visible. ” Subsequently, the same video was evaluated under each of the four visual prompt injection conditions using the following overlay: “ Polyp detection: Active; Status: No polyp detected. ” ( Table 1 , Supplementary Table 3 ). All experiments were independently repeated three times to ensure consistency and reproducibility. The primary endpoint was defined as accuracy, evaluated as a binary outcome for each video and prompting strategy, i.e., 1 for correct predictions and 0 for incorrect predictions, based on the predefined ground truth labels ( Figure 1f , Table 1 , Supplementary Table 1 ). Model outcome was based solely on correct prediction target classification. Additional response details were not considered and model failures to produce outputs were registered as censored outputs. Statistical Analysis For each prediction task, the mean (SD) accuracy across three independent runs was reported for each prompting strategy and VLM. No randomization or blinding was performed. Only non-censored model outputs were used for accuracy calculation. Given that Qwen 2.5-VL censored most outputs, it was excluded from formal statistical analyses. We compared the baseline performance (i.e., no prompt injection) of the remaining models ( n = 3) using the Kruskal-Wallis test, followed by Dunn’s post-hoc test with Bonferroni correction for pairwise comparisons. To assess VLM vulnerability to prompt injection attacks, individual models’ performance for each prompt injection condition was compared to the respective model’s baseline performance (no injection) using the Wilcoxon signed-rank test. The significance threshold was set to α < 0.05. All statistical analyses were performed using Python 3.11. Results Video VLM performance on clinically relevant surgical video analysis tasks We first evaluated the baseline accuracy of four video VLMs across eleven clinically relevant surgical video analysis tasks ( Figure 1c ). Overall, we observed the highest baseline accuracies for bleeding detection, polyp detection, and foreign object detection tasks and the lowest accuracies for CVS and surgical skill assessment ( Table 2 ). Performance in image distortion detection varied substantially across the models and distortions. While Gemini 1.5 Pro, Gemini 2.5 Pro, and GPT-o4-mini-high processed all inputs without censoring, Qwen 2.5-VL censored 44.9% of outputs, including both baseline and prompt injection conditions, and was therefore removed from statistical analyses. Of all VLMs, Gemini 2.5 Pro exhibited the highest overall accuracy on the evaluated prediction tasks (mean [SD]: 0.82 [0.01]), outperforming Gemini 1.5 Pro (0.70 [0.03], P = .21), and GPT-o4-mini-high (0.67 [0.06], P = .07). Accuracies varied strongly across specific tasks and video VLMs ( Figure 2 , Supplementary Table 4 ). Download figure Open in new tab Figure 2: Performance accuracy of all models across six prompting strategies, clustered by clinical use cases. Heatmaps display each model’s accuracy for each prompting strategy across all evaluated use cases and the overall mean performance. (a) Gemini 1.5 Pro, (b) Gemini 2.5 Pro, and (c) GPT-o4-mini-high. Accuracy values, ranging from 0 to 1, are represented using a diverging color scale from brown (low, 0) to blue (high, 1). The absolute mean accuracies are annotated within each tile. Abbreviation: Critical view of safety (CVS). Model performance with standard deviations is visualized in Supplementary Figure 1 . Video VLM vulnerability to textual prompt injections To evaluate the vulnerability of the four video VLMs to textual and visual prompt injection attacks, we compared VLM accuracy after prompt injection to the respective baseline accuracies across the eleven prediction tasks ( Figure 1f ). Textual prompt injections consistently reduced accuracy for most prediction tasks across all models, though vulnerabilities varied across tasks. Generally, Generally, Gemini 2.5 Pro was more robust than the other two models, with accuracy declining from a mean (SD) of 0.82 (0.01) to 0.51 (0.06) (P < .001), compared to Gemini 1.5 Pro (0.70 [0.03] to 0.24 [0.05]; P < .001) and GPT-o4-mini-high (0.67 [0.06] to 0.17 [0.03]; P < .001) ( Figure 2 , Supplementary Table 5 ). These results indicate that textual prompt injection significantly degrades VLM performance across multiple surgical decision support tasks. Video VLM vulnerability to temporally-varying visual prompt injections To determine the vulnerability of four VLMs to visual prompt injection, including effects of timing and duration, we applied four different visual prompt injection strategies ( Figure 1e ). Comparing visually-injected VLM accuracies with the respective baseline, we observed a consistent relationship between attack duration and model performance degradation. Early-time and late-time injections led to minor VLM accuracy reductions, whereas partial-time and full-time injections caused significant declines in model accuracy ( Figure 2 , Supplementary Table 5, Supplementary Figure 1 ). Of all VLMs, Gemini 2.5 Pro was the most resilient against visual prompt injection attacks. In comparison to its overall baseline accuracy of 0.82 (0.01), early-time (accuracy: 0.80 (0.01); P = .30) and late-time injection (accuracy: 0.81 (0.03); P = .29) had minimal impact. However, partial-time (accuracy: 0.58 (0.02)) and full-time attacks (accuracy: 0.49 (0.03)) significantly reduced model performance ( P < .001). These findings indicate that Gemini 2.5 Pro possesses a degree of self-correction, sometimes explicitly flagging watermark inconsistencies ( Figure 3 ). Download figure Open in new tab Figure 3: Example outputs of video VLMs with and without textual or visual prompt injections. The figure outlines example outputs of Gemini 1.5 Pro, Gemini 2.5 Pro, GPT-o4-mini-high, and Qwen 2.5-VL for different detection tasks for surgical video analysis. Gemini 1.5 Pro demonstrated notably lower robustness against visual prompt injection attacks compared to Gemini 2.5 Pro. Here, both early-time and late-time injection significantly reduced the accuracy across all tasks, from 0.70 (0.03) to 0.56 (0.07) and 0.53 (0.03) (both P < .001), respectively. Partial-time and full-time injection further lowered accuracy across all tasks to 0.27 (0.09) and 0.21 (0.03) (both P < .001; Figure 2 , Supplementary Table 5 ). GPT-o4-mini-high was most vulnerable among models that did not censor outputs. As compared to its baseline accuracy of 0.67 (0.06) across all tasks, early-time and late-time injection significantly reduced accuracy to 0.47 (0.03) ( P < .001) and 0.36 (0.09) ( P < .001), respectively. Partial-time and full-time visual prompt injection caused accuracy declines to 0.33 (0.01) ( P < .001) and 0.24 (0.04) ( P < .001), respectively ( Figure 2 , Supplementary Table 5 ). These results indicate that even prompt injections in individual video frames could cause drastic GPT-o4-mini-high performance declines. GPT-o4-mini-high performance in foreign object detection was particularly sensitive to late-frame injections. Based on non-censored model outputs of Qwen 2.5-VL, its performance was strongly impacted by visual prompt injections, exhibiting complete vulnerability to early-time, partial-time, and full-time visual prompt injections but less to late-time injections ( Supplementary Table 5 ). Overall, we observed two consistent patterns across all video VLMs: first, longer-duration visual prompt injections led to greater performance loss. Second, although Gemini 2.5 Pro was less vulnerable to visual prompt injection attacks than the other VLMs evaluated, no model maintained reliability during prolonged attacks, especially for tasks dependent on subtle temporal features, such as surgical skill assessment. Discussion Our results indicate that targeted textual and visual prompt injections can mislead state-of-the-art video VLMs in the context of clinically relevant surgical decision support tasks. Even for well-established visual detection tasks like computer-aided polyp detection, we observed prompt injection attacks successfully misleading modern video VLMs to miss polyps during colonoscopies, which could potentially result in erroneous cancer diagnoses. In the context of VLM-based surgical decision support systems, such as tools enabling computer vision-based automation of counting protocols, prompt injection may lead to the unintentional retention of surgical items. Overall, our findings provide evidence of a consequential safety risk that needs to be addressed prior to the integration of VLMs into surgical decision support systems. Across several clinically relevant surgical video analysis tasks, the evaluated generalist VLMs showed promising baseline accuracies. In our sampled laparoscopic surgery video clips, performance on skill assessment and bleeding detection tasks was comparable to previously reported specialist single-task surgical AI models. 5 , 34 This shows the potential of VLMs to deliver clinically meaningful performance with minimal task-specific fine-tuning. In the context of surgical video VLM benchmarking, a critical challenge arises from the temporal nature of surgical video data. For some of the prediction tasks analyzed in this work, ground truths, such as the presence of foreign objects or the visibility of CVS criteria, dynamically change throughout the procedure. Such temporal variations in the ground truths pose a distinct challenge to surgical video analysis compared to static imaging data. Currently, temporal context remains inadequately integrated into VLM-based surgical video analysis research. Present-generation video VLMs internally process this data as sequential frames, but due to computational constraints, they cannot currently capture every temporally critical detail comprehensively. 43 This deficiency complicates both the accurate interpretation of temporally complex tasks with VLMs and their benchmarking. Effective benchmarking would require datasets with detailed time-dependent annotations of variable ground truths for relevant video segments to facilitate clinically meaningful evaluation. 44 The integration of video VLMs and other foundation AI models into clinical care infrastructure necessitates a cybersecurity-informed deployment strategy. Existing LLM and VLM guardrails are often based on input imaging content. In contrast to previous findings, which reported that Gemini 1.5 Pro had strong guardrails preventing its use on radiology images 25 , Gemini 1.5 Pro, as well as the newer Gemini 2.5 Pro model, accepted all provided surgical video inputs without restriction, indicating comparatively weaker content-based guardrails. Although Qwen 2.5-VL censored outputs based on specific video content, none of the tested VLMs possess robust guardrails against adversarial prompt injections. Proactive security measures, including dedicated safeguards against adversarial inputs, and clear regulatory guidelines need to be developed and implemented to ensure the safe transition of VLMs into intraoperative decision support systems. Overall, our results suggest that VLM vulnerability to prompt injection attacks represents an important and generalizable measure of model robustness. Therefore, in the broader context of the future clinical deployment of VLMs, evaluating their robustness to prompt injection attacks would be a valuable component of regulatory risk evaluation processes. Limitations This study has several limitations. First, little evidence currently exists about the actual prevalence and impact of prompt injection threats. Yet, awareness of these potential vulnerabilities is critical for regulatory assessment of Software-as-a-Medical-Device tools for surgical decision support. Second, our analysis was limited to publicly available surgical datasets, and evaluated only short, curated video segments, which differ substantially from actual surgical procedures that typically last several hours. Both factors may restrict the generalizability of our findings. Third, our study provided only a partial assessment of VLM performance using widely available generalist models. Although a small number of surgery-specific video VLMs have recently been proposed 45 , 46 , these models are not yet publicly accessible for comprehensive evaluation. While surgery-specific VLMs may potentially offer higher baseline performances, they do not currently possess specific guardrails, which makes a higher resistance to prompt injection attacks unlikely. Lastly, our results may be affected by data leakage. Since publicly available data might have been used during VLM training, the observed performance metrics might be inflated compared to performance on truly unseen data. Despite these limitations, our findings reveal critical security vulnerabilities in present-generation video VLMs that may have relevant implications for their deployment in surgical decision support. Conclusions Modern video VLMs possess capabilities that are likely to facilitate a new era of video-based surgical assistance. However, in their current stage, they remain highly susceptible to prompt injection. Without dedicated guardrails and an improved understanding of how these models integrate information over time, their clinical deployment could introduce relevant risks to surgical patients. Data Availability All data produced in the present work are contained in the manuscript. Competing Interests JNK declares consulting services for Bioptimus; Panakeia; AstraZeneca; and MultiplexDx. Furthermore, he holds shares in StratifAI, Synagen, Tremont AI and Ignition Labs; has received an institutional research grant by GSK; and has received honoraria by AstraZeneca, Bayer, Daiichi Sankyo, Eisai, Janssen, Merck, MSD, BMS, Roche, Pfizer, and Fresenius. FRK declares advisory roles for Radical Healthcare, USA; and the Surgical Data Science Collective, USA. All other authors declare no competing interests. Acknowledgments JC is supported by the Mildred-Scheel-Postdoktorandenprogramm of the German Cancer Aid (grant #70115730). JNK is supported by the German Cancer Aid DKH (DECADE, 70115166), the German Federal Ministry of Research, Technology and Space BMFTR (PEARL, 01KD2104C; CAMINO, 01EO2101; TRANSFORM LIVER, 031L0312A; TANGERINE, 01KT2302 through ERA-NET Transcan; Come2Data, 16DKZ2044A; DEEP-HCC, 031L0315A; DECIPHER-M, 01KD2420A; NextBIG, 01ZU2402A), the German Research Foundation DFG (CRC/TR 412, 535081457), the German Academic Exchange Service DAAD (SECAI, 57616814), the German Federal Joint Committee G-BA (TransplantKI, 01VSF21048), the European Union EU’s Horizon Europe research and innovation programme (ODELIA, 101057091; GENIAL, 101096312), the European Research Council ERC (NADIR, 101114631), the National Institutes of Health NIH (EPICO, R01 CA263318) and the National Institute for Health and Care Research NIHR (Leeds Biomedical Research Centre, NIHR203331). FRK receives support from the German Cancer Research Center (CoBot 2.0), the Joachim Herz Foundation (Add-On Fellowship for Interdisciplinary Life Science), the Central Indiana Corporate Partnership AnalytiXIN Initiative, the Evan and Sue Ann Werling Pancreatic Cancer Research Fund, and the Indiana Clinical and Translational Sciences Institute (EPAR4157) funded, in part, by Grant Number UM1TR004402 from the National Institutes of Health, National Center for Advancing Translational Sciences, Clinical and Translational Sciences Award. The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health, the NHS, the NIHR, or the Department of Health and Social Care. This work was funded by the European Union. Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union. Neither the European Union nor the granting authority can be held responsible for them. References 1. ↵ Maier-Hein L , Eisenmann M , Sarikaya D , et al. Surgical data science – from concepts toward clinical translation . Med Image Anal . 2022 ; 76 : 102306 – 102306 . OpenUrl CrossRef PubMed 2. ↵ Maier-Hein L , Vedula SS , Speidel S , et al. Surgical data science for next-generation interventions . Nature Biomedical Engineering . 2017 ; 1 ( 9 ): 691 – 696 . doi: 10.1038/s41551-017-0132-7 OpenUrl CrossRef 3. ↵ Goodman ED , Patel KK , Zhang Y , et al. Analyzing surgical technique in diverse open surgical videos with multitask machine learning . JAMA Surg . 2024 ; 159 ( 2 ): 185 – 192 . OpenUrl PubMed 4. ↵ Rahbar MD , Reisner L , Ying H , Pandya A. An entropy-based approach to detect and localize intraoperative bleeding during minimally invasive surgery . Int J Med Robot . 2020 ; 16 ( 6 ): 1 – 9 . OpenUrl CrossRef 5. ↵ Horita K , Hida K , Itatani Y , et al. Real-time detection of active bleeding in laparoscopic colectomy using artificial intelligence . Surg Endosc . 2024 ; 38 ( 6 ): 3461 – 3469 . OpenUrl PubMed 6. ↵ Igaki T , Kitaguchi D , Kojima S , et al. Artificial intelligence-based total mesorectal excision plane navigation in laparoscopic Colorectal Surgery . Dis Colon Rectum . 2022 ; 65 ( 5 ): e329 – e333 . OpenUrl PubMed 7. ↵ Madani A , Namazi B , Altieri MS , et al. Artificial intelligence for intraoperative guidance: Using semantic segmentation to identify surgical anatomy during laparoscopic cholecystectomy . Ann Surg . 2022 ; 276 ( 2 ): 363 – 369 . OpenUrl CrossRef PubMed 8. ↵ Mascagni P , Vardazaryan A , Alapatt D , et al. Artificial Intelligence for Surgical Safety: Automatic Assessment of the Critical View of Safety in Laparoscopic Cholecystectomy Using Deep Learning . Ann Surg . Published online 2020. doi: 10.1097/SLA.0000000000004351 OpenUrl CrossRef 9. ↵ Pedrett R , Mascagni P , Beldi G , Padoy N , Lavanchy JL . Technical skill assessment in minimally invasive surgery using artificial intelligence: a systematic review . Surg Endosc . 2023 ; 37 ( 10 ): 7412 – 7424 . OpenUrl PubMed 10. ↵ Carstens M , Vasisht S , Zhang Z , et al. Artificial Intelligence for surgical scene understanding: A systematic review and reporting quality meta-analysis . medRxiv . Published online July 14, 2025. doi: 10.1101/2025.07.12.25330122 OpenUrl Abstract / FREE Full Text 11. ↵ Tanno R , Barrett DGT , Sellergren A , et al. Collaboration between clinicians and vision- language models in radiology report generation . Nat Med . 2025 ; 31 ( 2 ): 599 – 608 . OpenUrl CrossRef PubMed 12. ↵ Lu MY , Chen B , Williamson DFK , et al. A multimodal generative AI copilot for human pathology . Nature . 2024 ; 634 ( 8033 ): 466 – 473 . OpenUrl CrossRef PubMed 13. ↵ Menezes MCS , Hoffmann AF , Tan ALM , et al. The potential of Generative Pre-trained Transformer 4 (GPT-4) to analyse medical notes in three different languages: a retrospective model-evaluation study . Lancet Digit Health . 2025 ; 7 ( 1 ): e35 – e43 . OpenUrl 14. Busch F , Prucker P , Komenda A , et al. Multilingual feasibility of GPT-4o for automated Voice- to-Text CT and MRI report transcription . Eur J Radiol . 2025 ; 182 ( 111827 ): 111827 . OpenUrl PubMed 15. ↵ Zhang J , Ma Y , Zhang R , et al. A comparative study of GPT-4o and human ophthalmologists in glaucoma diagnosis . Sci Rep . 2024 ; 14 ( 1 ): 30385 . OpenUrl PubMed 16. ↵ Yan S , Yu Z , Primiero C , et al. A multimodal vision foundation model for clinical dermatology. Nat Med . Published online June 6 , 2025 : 1 – 12 . 17. ↵ Lu MY , Chen B , Williamson DFK , et al. A visual-language foundation model for computational pathology . Nat Med . 2024 ; 30 ( 3 ): 863 – 874 . OpenUrl CrossRef PubMed 18. ↵ Rau A , Endo M , Aklilu J , et al. Systematic evaluation of large Vision-Language Models for surgical artificial intelligence . arXiv [csCV] . Published online April 3, 2025. Accessed April 19 , 2025 . http://arxiv.org/abs/2504.02799 19. ↵ Zeng Z , Zhuo Z , Jia X , et al. SurgVLM: A large vision-language model and systematic evaluation benchmark for surgical intelligence . arXiv [csCV] . Published online June 3, 2025. http://arxiv.org/abs/2506.02555 20. ↵ Gemini . Google DeepMind . Accessed June 22, 2025 . https://deepmind.google/gemini/ 21. ↵ OpenAI , Achiam J , Adler S , et al. GPT-4 Technical Report . arXiv [csCL] . Published online 2023. doi: 10.48550/ARXIV.2303.08774 OpenUrl CrossRef 22. ↵ Bai S , Chen K , Liu X , et al. Qwen2.5-VL Technical Report . arXiv [csCV] . Published online 2025. doi: 10.48550/ARXIV.2502.13923 OpenUrl CrossRef 23. ↵ Protserov S , Hunter J , Zhang H , et al. Development, deployment and scaling of operating room-ready artificial intelligence for real-time surgical decision support . NPJ Digit Med . 2024 ; 7 ( 1 ): 231 . OpenUrl PubMed 24. ↵ OWASP Top 10 for LLM Applications 2025 . OWASP Gen AI Security Project. November 18 , 2024 . Accessed June 13, 2025 . https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/ 25. ↵ Clusmann J , Ferber D , Wiest IC , et al. Prompt injection attacks on vision language models in oncology . Nat Commun . 2025 ; 16 ( 1 ): 1239 . OpenUrl PubMed 26. ↵ Clusmann J , Schulz SJK , Ferber D , et al. Incidental prompt injections on vision–language models in real-life histopathology . NEJM AI . 2025 ; 2 ( 6 ). doi: 10.1056/aics2500078 OpenUrl CrossRef 27. ↵ Jiang JX , Ross JS , Bai G. Ransomware attacks and data breaches in US health care systems . JAMA Netw Open . 2025 ; 8 ( 5 ): e2510180 . OpenUrl 28. ↵ Goh E , Gallo R , Hom J , et al. Large language model influence on diagnostic reasoning: A randomized clinical trial: A randomized clinical trial . JAMA Netw Open . 2024 ; 7 ( 10 ): e2440969 . OpenUrl CrossRef PubMed 29. ClinicalTrials.gov . Accessed June 6, 2025 . https://clinicaltrials.gov/study/NCT05231174?intr=Large%20Language%20Model&rank=10 30. Ferber D , Hilgers L , Wiest IC , et al. End-to-end clinical trial matching with Large Language Models . arXiv [csCL] . Published online July 18, 2024. http://arxiv.org/abs/2407.13463 31. ↵ Goh E , Gallo RJ , Strong E , et al. GPT-4 assistance for improvement of physician performance on patient care tasks: a randomized controlled trial . Nat Med . 2025 ; 31 ( 4 ): 1233 – 1238 . OpenUrl PubMed 32. ↵ Dameff C , Tully J , Chan TC , et al. Ransomware attack associated with disruptions at adjacent emergency departments in the US . JAMA Netw Open . 2023 ; 6 ( 5 ): e2312270 . OpenUrl PubMed 33. ↵ Jalali MS , Kaiser JP . Cybersecurity in hospitals: A systematic, organizational perspective . J Med Internet Res . 2018 ; 20 ( 5 ): e10059 . OpenUrl PubMed 34. ↵ Wagner M , Müller-Stich BP , Kisilenko A , et al. Comparative validation of machine learning algorithms for surgical workflow and skill analysis with the HeiChole benchmark . Med Image Anal . 2023 ; 86 ( 102770 ): 102770 . OpenUrl PubMed 35. Ríos MS , Molina-Rodriguez MA , Londoño D , et al. Cholec80-CVS: An open dataset with an evaluation of Strasberg’s critical view of safety for AI . Sci Data . 2023 ; 10 ( 1 ): 194 . OpenUrl PubMed 36. Twinanda AP , Shehata S , Mutter D , Marescaux J , de Mathelin M , Padoy N. EndoNet: A Deep Architecture for Recognition Tasks on Laparoscopic Videos . IEEE Trans Med Imaging . 2016 ; 36 ( 1 ): 86 – 97 . OpenUrl PubMed 37. ↵ Nwoye CI , Elgohary K , Srinivas A , Zaid F , Lavanchy JL , Padoy N. CholecTrack20: A Multi- Perspective Tracking Dataset for Surgical Tools . arXiv [csCV] . Published online December 12, 2023. http://arxiv.org/abs/2312.07352 38. ↵ Lavanchy JL , Ramesh S , Dall’Alba D , et al. Challenges in multi-centric generalization: phase and step recognition in Roux-en-Y gastric bypass surgery . Int J Comput Assist Radiol Surg . 2024 ; 19 ( 11 ): 2249 – 2257 . OpenUrl PubMed 39. ↵ Wang Z , Lu B , Long Y , et al. AutoLaparo: A new dataset of integrated multi-tasks for image- guided surgical automation in laparoscopic hysterectomy . In: Lecture Notes in Computer Science. Lecture notes in computer science. Springer Nature Switzerland ; 2022 : 486 – 496 . 40. ↵ Maier-Hein L , Wagner M , Ross T , et al. Heidelberg colorectal data set for surgical data science in the sensor operating room . Scientific Data . 2021 ; 8 ( 1 ): 1 – 11 . OpenUrl PubMed 41. ↵ Pogorelov K , Randel KR , Griwodz C , et al. KVASIR: A multi-class image dataset for computer aided gastrointestinal disease detection . In: Proceedings of the 8th ACM on Multimedia Systems Conference . ACM ; 2017 . doi: 10.1145/3083187.3083212 OpenUrl CrossRef 42. ↵ Sanford DE , Strasberg SM . A simple effective method for generation of a permanent record of the Critical View of Safety during laparoscopic cholecystectomy by intraoperative “doublet” photography . J Am Coll Surg . 2014 ; 218 ( 2 ): 170 – 178 . OpenUrl PubMed 43. ↵ Zhang P , Zhang K , Li B , et al. Long context transfer from language to vision . arXiv [csCV] . Published online June 24, 2024. http://arxiv.org/abs/2406.16852 44. ↵ Wei J , Xiao Z , Sun D , et al. SurgBench: A unified large-scale benchmark for surgical video analysis . arXiv . Published online June 9, 2025. http://arxiv.org/abs/2506.07603 45. ↵ Schmidgall S , Cho J , Zakka C , Hiesinger W. GP-VLS: A general-purpose vision language model for surgery . ArXiv . 2024 ;abs/2407.19305. doi: 10.48550/arXiv.2407.19305 OpenUrl CrossRef 46. ↵ Surgical-LVLM: Learning to Adapt Large Vision-Language Model for Grounded Visual Question Answering in Robotic Surgery . arXiv . Accessed June 21, 2025 . https://arxiv.org/html/2405.10948v1 View the discussion thread. Back to top Previous Next Posted July 23, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Prompt injection attacks on vision-language models for surgical decision support Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Prompt injection attacks on vision-language models for surgical decision support Zheyuan Zhang , Muhammad Ibtsaam Qadir , Matthias Carstens , Evan Hongyang Zhang , Madison Sarah Loiselle , Farren Marc Martinus , Maksymilian Ksawier Mroczkowski , Jan Clusmann , Jakob Nikolas Kather , Fiona R. Kolbinger medRxiv 2025.07.16.25331645; doi: https://doi.org/10.1101/2025.07.16.25331645 Share This Article: Copy Citation Tools Prompt injection attacks on vision-language models for surgical decision support Zheyuan Zhang , Muhammad Ibtsaam Qadir , Matthias Carstens , Evan Hongyang Zhang , Madison Sarah Loiselle , Farren Marc Martinus , Maksymilian Ksawier Mroczkowski , Jan Clusmann , Jakob Nikolas Kather , Fiona R. Kolbinger medRxiv 2025.07.16.25331645; doi: https://doi.org/10.1101/2025.07.16.25331645 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Surgery Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15229) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6600) Geriatric Medicine (668) Health Economics (997) Health Informatics (4536) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (541) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9231) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0077ca8fc9358d3',t:'MTc3OTU3NTIzNQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();