Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol

Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol View ORCID Profile Xiaoyi Ji , View ORCID Profile Renata Zelic , View ORCID Profile Oskar Aspegren , View ORCID Profile Nita Mulliqi , View ORCID Profile Michelangelo Fiorentino , View ORCID Profile Francesca Giunchi , Luca Molinaro , View ORCID Profile Sol Erika Boman , View ORCID Profile Kelvin Szolnoky , Luana Xuan Liu , Andreas Pettersson , View ORCID Profile Per Henrik Vincent , View ORCID Profile Martin Eklund , View ORCID Profile Olof Akre , View ORCID Profile Kimmo Kartasalo doi: https://doi.org/10.1101/2025.09.22.25336169 Xiaoyi Ji 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Xiaoyi Ji Renata Zelic 2 Department of Molecular Medicine and Surgery, Karolinska Institutet , Stockholm, Sweden 3 Department of Pelvic Cancer , Cancer Theme, Karolinska University Hospital , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Renata Zelic Oskar Aspegren 2 Department of Molecular Medicine and Surgery, Karolinska Institutet , Stockholm, Sweden 3 Department of Pelvic Cancer , Cancer Theme, Karolinska University Hospital , Stockholm, Sweden 4 Department of Pathology and Cancer Diagnostics, Karolinska University Hospital , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Oskar Aspegren Nita Mulliqi 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Nita Mulliqi Michelangelo Fiorentino 5 Department of Medical and Surgical Sciences, University of Bologna , Bologna, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Michelangelo Fiorentino Francesca Giunchi 6 Department of Pathology , IRCCS Azienda Ospedaliero-Universitaria di Bologna, Bologna, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Francesca Giunchi Luca Molinaro 7 Division of Pathology , AOU Città Della Salute e Della Scienza di Torino, Turin, Italy Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sol Erika Boman 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden 2 Department of Molecular Medicine and Surgery, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sol Erika Boman Kelvin Szolnoky 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kelvin Szolnoky Luana Xuan Liu 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site Andreas Pettersson 3 Department of Pelvic Cancer , Cancer Theme, Karolinska University Hospital , Stockholm, Sweden 8 Clinical Epidemiology Division, Department of Medicine Solna, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site Per Henrik Vincent 2 Department of Molecular Medicine and Surgery, Karolinska Institutet , Stockholm, Sweden 3 Department of Pelvic Cancer , Cancer Theme, Karolinska University Hospital , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Per Henrik Vincent Martin Eklund 1 Department of Medical Epidemiology and Biostatistics, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Martin Eklund Olof Akre 2 Department of Molecular Medicine and Surgery, Karolinska Institutet , Stockholm, Sweden 3 Department of Pelvic Cancer , Cancer Theme, Karolinska University Hospital , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Olof Akre Kimmo Kartasalo 9 Department of Medical Epidemiology and Biostatistics , SciLifeLab, Karolinska Institutet , Stockholm, Sweden Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kimmo Kartasalo For correspondence: kimmo.kartasalo{at}ki.se Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF ABSTRACT Introduction Prostate cancer diagnosis and treatment planning depend on accurate histopathological assessment of needle biopsies, particularly through the Gleason scoring system. The inherently subjective nature of the grading creates variability between pathologists, potentially resulting in suboptimal patient management decisions. These reproducibility challenges extend beyond Gleason scoring to encompass other critical diagnostic and prognostic markers, including cancer volume quantification and detection of cribriform morphology patterns and perineural invasion. Artificial intelligence (AI) applications in digital pathology have emerged as promising solutions for enhancing diagnostic consistency and accuracy, with recent research demonstrating that automated systems can match expert-level performance in prostate biopsy evaluation. Nevertheless, comprehensive validation studies have revealed concerning limitations in model generalisability when deployed across different clinical environments and patient populations. Recent systematic reviews revealed widespread risk-of-bias limitations and insufficient external validation in AI diagnostic studies, highlighting critical needs for accumulated evidence supporting generalisability before clinical implementation. Rigorous external validation with preregistered protocols using independent datasets from diverse clinical settings remains essential to establish the reliability and safety of AI-assisted prostate pathology systems. Methods and analysis This study protocol establishes a framework for the retrospective external validation of an AI system developed for prostate biopsy assessment, to be conducted on the case-control samples of the National Prostate Cancer Register of Sweden, ProMort study (1998-2015). The primary aim is to evaluate the AI model’s diagnostic accuracy and Gleason grading performance using completely independent datasets separate from any model development or previously used validation cohorts. The diversity of the validation samples, spanning multiple geographic regions, temporal collection periods, and reference standards, allows evaluation of model robustness across varied clinical contexts. Secondary aims encompass evaluating AI performance in cancer length estimation and detection of cribriform patterns and perineural invasion. This protocol delineates procedures for data collection, reference standard clarification, and prespecified statistical analyses, ensuring comprehensive validation and reliable performance assessment. The study design conforms to established reporting guidelines CLAIM and STARD-AI, and recognised best practices for AI validation in medical imaging. Ethics and dissemination Data collection and usage were approved by the Swedish Regional Ethics Review Board and the Swedish Ethical Review Authority (permits 2012/1586-31/1, 2016/613-31/2, 2019-01395, 2019-05220). The study adheres to the Declaration of Helsinki principles, and findings will be made available in open access peer-reviewed publications. STRENGTHS AND LIMITATIONS This study incorporates case-control subsamples from Sweden’s largest clinical prostate cancer database (the National Prostate Cancer Register, NPCR), capturing a broad spectrum of variation across Swedish regions. The validation dataset encompasses samples collected from 1998 to 2015, representing one of the first AI validation studies to systematically evaluate performance across such an extensive temporal range, capturing evolving histological sample preparation techniques and changing population characteristics. A consistent scanning and annotation platform during digitisation eliminates equipment-related technical variation, while standardised annotation protocols among pathologists ensure traceable and reliable reference standards. Case-control design with 50% cancer-related mortality may create spectrum and prevalence bias, limiting comparison with typical clinical populations and other AI studies. Differences between the diagnostic reporting guidelines applied to the AI model’s training data and our validation dataset may introduce systematic differences that affect the interpretation of AI-pathologist concordance measurements. INTRODUCTION Systematic histopathological assessment of prostate needle biopsies constitutes an essential component of cancer diagnosis and treatment stratification, fundamentally guiding patient management strategies. The architectural patterns-based Gleason scoring (GS) system, introduced by Donald Gleason et al. 1 , established the foundational approach for prostate cancer grading using primary and secondary pattern summation (e.g., 3 + 4 = 7) 2 , though its inherently subjective nature leads to significant inter- and intra-pathologist variability that places patients at risk of inappropriate treatment decisions 3 , 4 . To address these reproducibility concerns, the International Society of Urological Pathology (ISUP) conducted systematic consensus conferences in 2005, 2014, and 2019, progressively refining grading criteria. Notably, major changes to the pattern descriptions and elimination of patterns 1-2 took place in 2005 5 , followed by introducing the simplified 5-tier Grade Group system in 2014 to replace the complex Gleason score combinations 6 , and establishing refined quantitative assessment criteria in 2019 while including digital pathology workflows combined with artificial intelligence (AI) into consideration for future work 7 . Similar standardisation challenges have affected the assessment of other clinically relevant histopathological features critical for prognosis and treatment, including cribriform cancer morphology (associated with aggressive behaviour and poor outcomes) 8 and perineural invasion (PNI, a marker of extraprostatic extension risk) 9 . Additionally, the quantification of cancer extent in prostatic biopsies presents persistent methodological complexity, with recent international surveys revealing that pathologists employ remarkably diverse approaches 10 , 11 , including multiple techniques for linear measurements (measuring only the largest focus, summing all foci versus spanning from first to last cancer area). This results in substantial inter-observer variability that directly impacts active surveillance eligibility decisions. Despite decades of standardisation efforts through professional societies and evidence-based guideline development, significant diagnostic variability persists, highlighting the critical need for more objective and reproducible approaches to prostate cancer pathological evaluation. Recent advances in digital pathology and AI have demonstrated remarkable progress in automated prostate cancer diagnosis. Studies by Campanella et al. in 2019 12 , Ström et al. in 2020 13 , and Bulten et al. in 2020 14 established that AI systems can achieve pathologist-level diagnostic accuracy. The landmark PANDA Challenge, involving 1,290 developers analysing over 10,000 digitised biopsies, achieved agreement of 0.93 (Cohen’s quadratically weighted kappa, QWK) with expert uropathologists on internal validation 15 . These breakthroughs established a field that has produced multiple FDA-approved systems including Paige Prostate Detect, which in an evaluation of data from 218 institutions, enabled pathologists to reduce cancer detection errors by 70% compared with unassisted assessment 16 . The recent emergence of foundation models such as UNI 17 , Virchow 18 , 19 and CONCH 20 , which are trained on millions of histopathology images in a task-agnostic manner, now provides developers with general-purpose models with minimal fine-tuning requirements for diverse pathology tasks. However, systematic evidence reveals persistent generalisation challenges for AI-assisted prostate biopsy diagnosis. A review investigated 26 regulatory-cleared digital-pathology AI products, among which only 42% had a peer-reviewed external-validation publication 21 . A 2024 meta-analysis found that 99 out of 100 diagnostic-accuracy studies had at least one high or unclear risk-of-bias or applicability concern 22 . Performance commonly shows degradation on external cohorts (e.g., in PANDA the independent validations achieved agreements of 0.86 to 0.87, lower than the results on internal validation cohorts), with scanner and stain variability further degrading generalisation 23 , 24 . Substantial inter-observer variability in Gleason grading and differences in grading conventions introduce label noise that can hinder model transferability 25 , 26 . Moreover, temporal domain shifts caused by ageing archived specimens remain largely unexplored in current validation frameworks. This is an important gap, as prognostic applications often rely on historical material spanning decades, and such shifts may substantially affect AI performance across different eras of pathology practice 27 . Given these challenges, safe clinical deployment of AI-assisted histopathological diagnosis requires robust pre-deployment evaluation and systematic monitoring in use 28 . Based on this consideration, we propose an additional retrospective external validation of an in-house, tissue-specific AI system for prostate biopsy assessment, trained end-to-end with the attention-based multiple-instance learning mechanism for Gleason score classification 29 . The system was developed on 61,483 WSIs from 4,467 patients across four European sites, spanning multiple laboratories, scanners, and specimen characteristics 30 . The results showed that for prostate cancer grading, this task-specific model achieved comparable or superior performance to pipelines based on histopathological foundation models when trained or fine-tuned on the same data. To promote transparency and reproducibility, here we present a pre-specified protocol for retrospective external validation of this AI-assisted prostate cancer diagnosis system on the case-control samples of the National Prostate Cancer Register of Sweden (NPCR), ProMort study (1998-2015) 31 , 32 . We detail objectives, cohort definitions, and analysis pipelines to quantify agreement on key diagnostic outputs and to test AI model robustness against major sources of variability. This set-up limits analytic flexibility, reduces post-hoc bias, and provides an auditable record of the validation workflow to support subsequent clinical evaluation, with the primary aim to technically validate AI performance and to emphasise that caution should be exercised when extrapolating findings to clinical practice. Robust validation frameworks have emerged to address the rigorous evaluation requirements for AI-based diagnostic and prediction systems in healthcare. CLAIM (Checklist for Artificial Intelligence in Medical Imaging) 33 , updated in 2024, provides specialised reporting requirements for medical imaging AI studies, offering structured guidance covering the complete research pipeline from data acquisition to clinical implementation. The guideline emphasises transparent reporting of image acquisition protocols, reference standard definitions, and model evaluation procedures. The STARD-AI (Standards for Reporting Diagnostic Accuracy Studies using Artificial Intelligence) guideline 34 , updated in 2025, extends the original STARD 2015 framework 35 with 18 new or modified items specifically addressing AI-centered diagnostic test accuracy studies. The framework was developed through a multistage, multistakeholder consensus process and provides comprehensive reporting recommendations for studies evaluating the diagnostic accuracy of AI-based tests. STARD-AI places particular emphasis on dataset practices including data sources, annotation procedures, and partitioning strategies, while addressing critical considerations of algorithmic bias and fairness assessment. The guideline encourages detailed reporting of the AI index test evaluation processes, reference standard methodology, and transparent documentation of model performance across different demographic subgroups to ensure equitable healthcare delivery. This retrospective AI validation study adheres to these established reporting frameworks to ensure transparent documentation of data collection procedures, clear definition of reference standard, comprehensive evaluation of AI system performance and robustness, and explicit acknowledgement of study limitations and generalisability considerations. METHODS AND ANALYSIS Study objectives The primary objective of this study is to: Evaluate agreement between the AI model and pathologists in identifying prostate cancer and assigning Gleason scores in core needle biopsies of the prostate. In addition, the study addresses three secondary objectives: Evaluate the agreement between the AI model and pathologists in measuring the linear extent of cancer (in millimetres) within prostate core needle biopsies. Assess the agreement between the AI model and pathologists in identifying PNI in prostate core needle biopsies. Assess the agreement between the AI model and pathologists in detecting cribriform patterns of cancer in prostate core needle biopsies. Clinical implementation, user interaction, and the system’s performance when combined with human pathologists are beyond the scope of this study. Artificial intelligence system The AI system to be validated in this study was designed for the histopathological assessment of digitised prostate core needle biopsies 29 . Based on deep neural networks, the system incorporates specific image preprocessing steps, model architecture, and training strategies which were previously optimised during its development phase. The development and initial validation of the model followed a protocol-based approach 30 . System input The system accepts whole slide images (WSIs) in compatible vendor-specific formats, representing formalin-fixed, paraffin-embedded (FFPE) prostate core needle biopsy specimens stained with haematoxylin and eosin (H&E). Each image may contain one or more tissue sections from one or multiple biopsy cores. System output The raw output of the AI system consists of two probability vectors corresponding to the predicted primary and secondary Gleason patterns. Each vector contains four elements, representing the estimated probabilities for the following categories: benign, and Gleason grades 3 through 5. Gleason score: The AI system assigns a GS, such as 3 + 5 = 8, reflecting the primary and secondary histological patterns identified in the input WSI. The score is derived by selecting the most probable Gleason grade from each of the two prediction vectors. Possible scores range from 3 + 3 = 6 (least aggressive) to 5 + 5 = 10 (most aggressive), with 0 + 0 representing benign samples. ISUP grade: The assigned GS is further mapped to an ISUP grade from 1 to 5, which categorises cancer aggressiveness on an ordinal scale: ISUP 1 (GS 6), ISUP 2 (GS 3 + 4 = 7), ISUP 3 (GS 4 + 3 = 7), ISUP 4 (GS 8), and ISUP 5 (GS 9 - 10). Benign cases are assigned an ISUP grade of 0. Cancer diagnosis: A binary outcome is determined from the predicted ISUP grade, with samples assigned an ISUP grade > 0 classified as cancer-positive by the system. In addition, the system outputs a continuous probability estimate of cancer presence, calculated as one minus the minimum predicted probability of the benign class across the two Gleason pattern outputs. Cancer extent: The system will estimate the cancer length within the WSI in millimetres, providing a quantitative measure of the tumour’s spatial extent in the tissue. Cribriform cancer: A probability predicting the presence of cribriform morphology within the sample will be reported. Perineural invasion: A probability predicting the presence of PNI within the sample will be reported. Visualisation: Prediction results can be presented as visual overlays on the WSI, indicating regions associated with specific Gleason patterns, cribriform morphology, and PNI, with the exact appearance and format determined by subsequent processing. Study design In this study, we will perform a fully external validation of the diagnostic performance of the AI system, using retrospectively collected prostate biopsy data from a sample of Swedish men diagnosed with prostate cancer from 1998 to 2015 as a part of the ProMort I and ProMort II studies. The dataset represents a heterogeneous clinical environment, comprising patients from diverse backgrounds and clinical stages from multiple regions across Sweden, drawn from individuals whose samples were not involved in any phase of model development. The samples were digitised with a scanner of a different model (3DHistech) than the AI training data (Aperio, Hamamatsu, Philips) and prepared in laboratories not included in the development set. This introduces a domain shift in sample preparation and image acquisition characteristics, thereby providing a test of the AI system’s generalisability to previously unseen laboratories and scanner hardware. Interobserver variability among pathologists introduces heterogeneity in the reference standards within the validation dataset, as different pathologists independently annotated overlapping but non-identical subsets of cases. Because sample assignments were not stratified by cohort or region, variation in annotations arises in a non-systematic manner, which complicates the interpretation of the AI system’s performance. Observed differences in performance may thus reflect not only the model’s ability to generalise across diverse clinical data, but also inconsistencies in the human reference annotations. These two sources of variation, model generalisation and annotation variability, are difficult to disentangle and cannot be quantified. To address this challenge, we designed the validation study using two complementary strategies. First, we selected the annotations from a single pathologist, whose assessments cover the majority of the validation samples, as the primary reference standard. This enables consistent benchmarking across clinical sites and supports the evaluation of technical generalisation performance without confounding effects from interobserver variation, sample origin, or collection time. Second, for subsets of the ProMort I and ProMort II cohorts that were independently reviewed by multiple pathologists, we explicitly compared interobserver variability with AI-pathologist agreement. This allows us to contextualise the AI system’s performance relative to human variability and assess whether the AI demonstrates a comparable or higher level of consistency across diverse clinical settings. To ensure the integrity of the evaluation, the model development and validation phases were strictly separated to prevent any risk of information leakage. The study utilises digitised images of core needle biopsies from prostate cancer patients where both the AI system and human pathologists independently assessed the biopsy samples, with no access to each other’s results, guaranteeing a blinded evaluation process. No adjustments or modifications to any model parameters or settings were conducted during the validation procedure. The retrospective nature of this validation study does not inherently affect the validity of agreement metrics between AI and pathologists. However, retrospective designs lack real-time diagnostic feedback, precluding additional investigations (e.g., step-sections, immunohistochemistry) that would typically be employed to resolve diagnostic ambiguities in clinical practice. While cases with substantial AI-pathologist discrepancies are planned to undergo independent expert re-evaluation, this assessment remains constrained to existing histological material, potentially limiting the interpretation of apparent AI ‘errors’ that might be resolved through additional clinical workup. Clinical and pathological characteristics of the ProMort cohorts are presented in Tables 1 and 2 . Annotation coverage by each pathologist for diagnostic parameters is also summarised for Gleason grades and cancer length in Table 3 , and for cribriform morphology and PNI in Table 4 . These pathologist annotations constitute the reference standards for AI performance evaluation. View this table: View inline View popup Table 1. Patient-level clinical and pathological characteristics of ProMort I sample at the time of diagnosis from NPCR Data are presented for: 1) all patients from the original case-control study design (N=3,420), 2) patients with successfully scanned tissue biopsies (N=2,291), and 3) patients included for AI system validation after applying exclusion criteria on the subjects re-reviewed in 2017 (N=290). All clinical and pathological information is derived from the original NPCR database, with “Missing” indicating that information on variables recorded at diagnosis was not available in the underlying cohort. Abbreviations: SD, standard deviation; IQR, interquartile range; PSA, Prostate-Specific Antigen. View this table: View inline View popup Table 2. Patient-level clinical and pathological characteristics of ProMort II sample at the time of diagnosis from NPCR Data are presented for: 1) all patients from the original case-control study design (N=1,000), 2) patients with successfully scanned tissue biopsies (N=834, including 4 duplicate controls that were subsequently removed), and 3) patients included for AI system validation after applying exclusion criteria on the subjects re-reviewed in 2019 (N=749). All clinical and pathological information is derived from the original NPCR database, with “Missing” indicating that information on variables recorded at diagnosis was not available in the underlying cohort. Abbreviations: SD, standard deviation; IQR, interquartile range; PSA, Prostate-Specific Antigen; LUTS, Lower urinary tract symptoms. View this table: View inline View popup Download powerpoint Table 3. Reference standard protocols with respect to Gleason grading and cancer length estimation for cases for AI model validation ProMort II comprises two subsamples: case-control and software validation. All annotations were performed at the core level, including both cancer-positive and cancer-negative cores. In the Reviewer column, each row represents an individual pathologist with independent assessments in a blinded manner. For each outcome measure, shaded regions indicate annotation coverage by the respective pathologists, with vertically overlapping shaded regions representing cores reviewed by multiple pathologists. For ProMort I, cores reviewed by F.G. and M.F. only (excluding L.M.) represent the ones rejected by L.M. during his review phase. The relative lengths of the shaded bars are illustrative only and do not reflect the exact proportions of annotation coverage. View this table: View inline View popup Download powerpoint Table 4. Reference standard protocols with respect to cribriform pattern and PNI detection for cases for AI model validation ProMort II comprises two subsamples: case-control and software validation. All annotations were performed at the focus region level, and aggregated to core level for validation. In the Focus region number column, the counts include only the regions with tumours. In the Reviewer column, each row represents an individual pathologist with independent assessments in a blinded manner. For each outcome measure, shaded regions indicate annotation coverage by respective pathologists at the core level, with vertically overlapping shaded regions representing cores reviewed by multiple pathologists. The relative lengths of the shaded bars are illustrative only and do not reflect the exact proportions of annotation coverage. Data sources ProMort I Source ProMort I is a nested case-control study derived from NPCR 31 . Men diagnosed with low- to intermediate-risk prostate cancer between January 1, 1998, and December 31, 2011, were eligible for inclusion. Risk stratification criteria included: clinical stage T1-T2, GS ≤7 or World Health Organization (WHO) grade 1 if GS unavailable, serum Prostate-Specific Antigen (PSA) <20 ng/mL, and absence of nodal (N0/Nx) or distant (M0/Mx) metastases. From approximately 58,000 eligible men, 1,735 prostate cancer deaths were identified through December 31, 2012. Controls were selected through incidence-density sampling, matched 1:1 on year and institution of diagnosis. After excluding 25 cases lacking eligible controls, the final cohort comprised 1,710 matched case-control pairs. Slide digitisation and histopathological review Diagnostic specimens were retrieved from pathology departments throughout Sweden and digitised at Örebro University Hospital between November 2015 and February 2016 using a Pannoramic 250 Flash II scanner (3DHistech Ltd., Budapest, Hungary) at 40× magnification (0.19 µm/pixel resolution). From 3,420 sampled patients, slides were successfully retrieved and scanned for 2,290 patients (67%), yielding 14,036 digital images stored in MRXS format. A subset of 356 patients from Örebro (n=44) and Skåne (n=312) counties was selected as a pathological re-review subsample to confirm low- to intermediate-risk classification and establish interobserver concordance. The review was conducted between June 2017 and April 2018, with annotation terminated after 313 patients (44 from Örebro, 269 from Skåne; 159 cases, 154 controls) based on interim assessment of data adequacy. Reference standard protocol Digital assessment was performed using a virtual microscopy system developed in collaboration between the Centre for Advanced Studies, Research and Development in Sardinia (CRS4), Pula, Italy, and the ProMort study 36 . Three genitourinary pathologists (F.G., L.M., and M.F. with 8, 4, and 19 years of experience in genitourinary pathology at the time of the review, respectively) evaluated cases following the 2014 ISUP modified Gleason grading system. The pathologists were blinded to the original clinical and histopathological information and the case-control status. The annotation followed a structured workflow: Slide-level screening : Firstly, F.G. systematically reviewed all slides for each patient, first determining whether slides should be excluded by rejecting duplicate slides, slides lacking prostatic tissue, or specimens other than core-needle biopsies (e.g., TURP specimens, lymph node specimens). For eligible slides, F.G. identified all tissues at the core level with spatial delineation. To avoid redundancy, when cores were represented by multiple slices, only the most representative slice was selected for subsequent scoring. Core and focus region annotation: F.G. performed comprehensive annotation including cancer detection, tumour length measurement, and Gleason grading for each core. For cancer positive cores, all areas with cancer were marked as separate focus regions with spatial delineation, plus one selected area of normal tissue per core. Each positive focus region underwent detailed prognostic feature assessment, including perineural involvement and cribriform pattern. Then L.M. independently reviewed all cancer-positive cores annotated by F.G. for Gleason grading, and additionally assessed prognostic features for all positive focus regions within a randomly selected subset of these cores. Cases with interobserver disagreement on Gleason grading underwent adjudication by M.F., who reviewed both Gleason grading and prognostic features for positive focus regions within the discordant cores. Ultimately, 290 patients (146 cases and 144 controls) were used for the AI model validation. Detailed description of the ProMort I inclusion and exclusions is presented in a flow-chart in Figure 1 . Population characteristics at the time of diagnosis are summarised in Table 1 . The annotation protocol is available in Supplementary Material 1. Download figure Open in new tab Figure 1. Flow chart of population selection for ProMort I sample, its re-review subsample and AI validation subsample N represents the number of patients. Abbreviations: PCa, prostate cancer; NPCR, National Prostate Cancer Register of Sweden; PSA, prostate-specific antigen; T, tumour; N, nodes; M, metastasis; GS, Gleason score; PCSM, prostate cancer specific mortality; H&E, hematoxylin and eosin. ProMort II Source ProMort II is a case-control sample of men in NPCR diagnosed with non-metastatic (M0/Mx) prostate cancer between January 1, 1998, and December 31, 2015 32 . The sample was selected from 11 of 21 Swedish counties, chosen based on their likelihood of providing slides for digitisation. The case-control sample comprised 500 cases (random sample of men who died of prostate cancer) and 500 controls (sampled with replacement by incidence density sampling), matched 1:1 on year and county of diagnosis. Slide digitisation and histopathological review Biopsy slides were retrieved from pathology departments and scanned at Örebro University Hospital using the same scanner and scanning protocol as ProMort I. The scanning took place between May 2017 and January 2018, approximately 2-20 years after slide preparation. From the 1,000 sampled patients, slides were successfully retrieved and scanned for 830 patients (83%), yielding 5,536 digital images stored in MRXS format. Two complementary subsamples were then defined: Software validation subsample (n=60): A random sample of 26 cases and 34 controls from two counties (Örebro n=25, Värmland n=35) selected for multi-pathologist annotation, with the original purpose to assess interobserver variability based on different microscopy setups. 25 Case-control subsample (n=714): The remaining subjects with evaluable slides after extraction of the software validation set, with 359 cases and 355 controls. Reference standard protocol Digital assessment utilised the same virtual microscopy platform as described for ProMort I. Three genitourinary pathologists (F.G., M.F. and Os.A. with 10, 21 and 2 years of experience at the time of the review, respectively) were involved in independently reviewing the scanned images following the 2016 WHO Classification of Tumours of the Urinary System and Male Genital Organs. The pathologists were blinded to the original clinical and histopathological information and the case-control status. The annotation workflow followed the same structured approach as ProMort I, consisting of slide-level screening and core and focus region annotation. However, unlike ProMort I where slide-level screening and spatial delineations were performed by a single pathologist (F.G.), each pathologist in ProMort II independently performed their own slide-level screening and delineated their own cores and focus regions for annotation. For each slide assigned to a pathologist for annotation, the reviewer first determined whether it should be excluded by rejecting duplicate slides, slides lacking prostatic tissue, or specimens other than core-needle biopsies (e.g., TURP specimens, lymph node specimens). To enable fair inter-observer comparisons at the core level, annotations were subsequently aligned to match corresponding anatomical regions across different pathologists’ markings. Case-control subsample : Each core was reviewed only once by a single pathologist (F.G. or M.F.), including cancer detection, tumour length measurement, Gleason grading and prognostic feature assessment, including perineural involvement and cribriform pattern. Software validation subsample : Each core was first reviewed three times: twice by F.G. and once by M.F. In 2020, Os.A. reviewed all 60 patients, applying a workflow following the same annotation protocol as F.G. and M.F. In total, 747 patients, of which 351 cases and 342 controls were in the case-control subsample and 22 cases and 32 controls were in the software validation subsample, were used for the AI model validation. Details of the ProMort II inclusion and exclusions are described in a flow-chart in Figure 2 and a summary of the population characteristics at the time of diagnosis is presented in Table 2 . Annotation protocols are provided in Supplementary Material 2 (case-control subsample) and Supplementary Material 3 (software validation subsample). Download figure Open in new tab Figure 2. Flow chart of population selection for ProMort II sample, its re-review subsample and AI validation subsamples N represents the number of patients. ¹ Based on the data extracted from NPCR June 5, 2020, but restricted to match conditions on April 11, 2017, when ProMort II was sampled. Abbreviations: PCa, prostate cancer; NPCR, National Prostate Cancer Register of Sweden; PCSM, prostate cancer specific mortality; H&E, hematoxylin and eosin. Definition of reference standard The reference standard consists of pathologists’ diagnoses at two levels: core-level and focus region-level. These annotations guide the evaluation of the AI system’s agreement with pathologists in diagnosing and grading prostate cancer in core needle biopsies. Core-level outcomes Core-level outcomes refer to the review of the individual biopsy cores, i.e. cylindrical tissue specimens, taken from each patient. The primary core-level outcomes are based on the pathologists’ annotations of the following features: Gleason score: The GS for a malignant individual biopsy core is determined by the most prevalent Gleason grade (primary grade) and the second most prevalent Gleason grade (secondary grade). However, in accordance with contemporary grading standards (2014 ISUP guidelines), any high-grade pattern (Gleason 4 or 5) was designated as the secondary pattern regardless of extent. Cancer length: The tumour extent of each core is measured as the total length of cancer, reported in millimetres. This measurement includes any intervening benign or non-invasive tissue, empty spaces between core fragments, and all cancerous foci, regardless of their separation. To ensure alignment across reviewers, cores within each slide were required to be annotated in a consistent order (left to right, top to bottom) so that core labels by different reviewers would generally correspond to each other. Focus region-level outcomes Focus regions refer to distinct tumour areas within malignant cores. For each identified focus region, pathologists annotated the high-risk features including: Cribriform pattern: The identification (presence or absence) of cribriform cancer morphology. Perineural invasion: The identification (presence or absence) of PNI. To ensure alignment across reviewers, focus regions within each core were required to be annotated in a consistent order (left to right, top to bottom) so that region labels by different reviewers would generally correspond to each other. Minor inconsistencies may occur due to inter-observer variation in defining the boundaries of individual regions. Spatial annotations All core-level and focus region-level annotations are accompanied by corresponding spatial delineations on the WSIs. Each annotated region is demarcated by a closed polygon defined by a series of coordinates, enabling precise localisation of the annotated areas. For Gleason score 7 cores, grade 4 areas were additionally delineated to enable detailed pattern analysis. These spatial annotations ensure reproducible identification of the exact tissue regions assessed by the pathologists and enable comparison between pathologist and AI assessments on the same tissue areas. Pre-analysis exclusion criteria Prior to AI validation analyses, all WSIs underwent standardised screening and quality control to ensure data integrity and compatibility with the AI system. The exclusion criteria included: Slides not stained by H&E according to the staining information recorded during the histopathological review step. This criterion resulted in the exclusion of 11 cases and 9 controls from ProMort I Skåne & Örebro subsample; 7 cases and 9 controls from ProMort II case-control subsample; and 4 cases and 2 controls from ProMort II software validation subsample. Slides that were rejected during the pathologist clinical review but erroneously retained his/her diagnostic annotations in the database. This resulted in the exclusion of 1 case and 4 controls from the ProMort II case-control subsample. Application of these exclusion criteria yielded the final AI validation datasets: ProMort I Skåne & Örebro subsample (N=290): 146 cases and 144 controls ProMort II: Case-control subsample (N=693): 351 cases and 342 controls Software validation subsample (N=54): 22 cases and 32 controls Data independence The population of ProMort I and II is fully external to the AI system, originating from different clinical sites and laboratories than those used during AI training, and scanned on a scanner not involved in the collection of training data. While there is some temporal overlap between the datasets (e.g. between ProMort and the 2012-2015 STHLM3 cohort used for AI training), the geographic separation of the populations ensures that the training and testing data remain independent. Statistical analyses Overview of statistical analyses All primary and secondary analyses follow a three-part validation framework: Validation against the consistent reference standard Subgroup analysis: Evaluate performance across different geographic regions Sensitivity analyses: Assess performance across multiple equally sized time intervals based on sample collection dates Assess performance against alternative reference standards, using subsets annotated by different pathologists The framework above will be applied to: Primary analysis : Cancer diagnosis and Gleason grading Secondary analysis : Cancer length prediction Secondary analysis : Cribriform cancer detection Secondary analysis : Perineural invasion detection Analyses I and II will be performed on samples annotated by the primary reference pathologist (F.G.), who reviewed the majority of cases in every validation subsample (see Table 3 and 4 for details). Details of statistical analyses The AI system performance will be evaluated at the core level for cancer diagnosis, cancer grading, and cancer length estimation. For detection of cribriform pattern and perineural invasion, performance will be evaluated at core level using reference standards aggregated from focus region annotations. The cores in which the AI system pre-processing fails to detect any tissue will be reported and excluded from any subsequent analyses. Grading and cancer extent analyses will be performed using two approaches: 1) including all cores (benign and malignant), and 2) restricting to cores with malignant diagnoses by the reference pathologist. Cancer detection analysis will include all cores, while PNI and cribriform analyses will be performed on malignant cores only. To quantify uncertainty around all performance estimates, 95% confidence intervals (CI) will be calculated using non-parametric bootstrap resampling clustered on patients (n=1,000 iterations). The evaluation metrics are summarised in Table 5 . View this table: View inline View popup Download powerpoint Table 5. Summary of evaluation metrics for primary and secondary analyses Metrics are categorised by their evaluation purpose: agreement (concordance between AI predictions and reference standard), discrimination (ability to distinguish between outcome categories), and calibration (alignment between predicted probabilities and observed frequencies). All metrics will be reported with 95% confidence intervals calculated using non-parametric bootstrap resampling (n=1,000 iterations). Abbreviations: AUC-ROC, area under the receiver operating characteristic curve; C-index, concordance index; MAE, mean absolute error; RMSE, root mean squared error; PPV, positive predictive value; NPV, negative predictive value. Primary analysis: Diagnosis and Gleason grading The concordance between the AI system’s outputs, including cancer diagnosis (positive/negative), GS, and ISUP grade, and the corresponding reference standards will be quantified. Annotations by the pathologist F.G. will serve as the primary reference standard and cores annotated by other pathologists will be used only for sensitivity analysis. For the ProMort II software validation subsample, where F.G. performed annotations twice independently, the first round of annotations will serve as the reference standard, with the intra-observer agreement between the two rounds previously reported 25 . Cancer diagnosis For the binary classification between benign and malignant cores, we will calculate sensitivity (true positive rate) and specificity (true negative rate). The AI system’s cancer probability output will be evaluated using area under the receiver operating characteristic curve (AUC-ROC) to assess discriminative ability and using calibration plots to examine the alignment between predicted probability of outcome and observed outcomes. Gleason score/ISUP grade To quantify agreement between AI predictions and reference standard for the grading systems, we will employ quadratically weighted kappa (QWK) as the primary measure, with linearly weighted kappa (LWK) and confusion matrices reported additionally. Ordinal concordance index (C-index) will be reported to measure model discrimination 37 . For GS, patterns will be encoded as ordinal variables: benign (0), 3+3 (1), 3+4 (2), 4+3 (3), 3+5 (4), 4+4 (5), 5+3 (6), 4+5 (7), 5+4 (8), 5+5 (9). ISUP grades naturally form an ordinal scale (0-5), with grade 0 representing benign tissue. Subgroup analysis: Evaluate performance across different geographic regions Cancer diagnosis and grading performance metrics will be computed using geographical stratification, which will be conducted at the county level for all Swedish regions except Skåne. Given the large sample size from Skåne county, further stratification at the municipal level will be implemented for this region. This analysis accounts for potential batch effects arising from inter-institutional variability in slide preparation protocols, including tissue processing and staining procedures. Sensitivity analysis: Evaluate performance across sample collection dates Analysis will be stratified by sample collection date by grouping the cores into multiple bins of equal time duration. To ensure unbiased performance comparisons across time periods, stratified sampling will be employed within each temporal bin to achieve comparable ISUP grade distributions. This approach allows assessment of whether temporal factors, such as evolving laboratory practices or tissue preservation methods, influence model grading performance. Sensitivity analysis: Assess performance against alternative reference standards We will quantify all-against-all pairwise agreements in panels comprising pathologists and the AI system to establish whether AI vs. reference standard discrepancies fall within the range of interobserver variation typically observed among pathologists. For the ProMort I Skåne & Örebro subsample, a subset of 548 cores was annotated by three pathologists (F.G., L.M., M.F.). For the ProMort II software validation subsample, a subset of 349 cores was annotated by three pathologists (F.G., M.F., Os.A.). For both subsets, pairwise agreement calculations will be conducted, enabling six comparisons per subset (three AI-pathologist and three pathologist-pathologist). The average AI-pathologist agreement will be compared to the average pathologist-pathologist agreement to determine whether the AI system achieves expert-level performance. Secondary analysis: Cancer length prediction We will quantify the concordance between the AI system’s prediction of linear cancer extent (in millimetres) and the reference standards at the core level. Annotations by the pathologist F.G. will serve as the primary reference standard and cores annotated by other pathologists will only be used for sensitivity analysis. Performance metrics: Concordance will be assessed using root mean squared error (RMSE), mean absolute error (MAE) and Bland-Altman limits of agreement as the primary measures, with Pearson’s correlation coefficient reported additionally. Scatter plots will visualise the relationship between predicted and reference standard cancer lengths. Subgroup analysis: Geographical stratification will be conducted as described for the primary grading analysis, with RMSE, MAE, Bland-Altman limits and correlation coefficients computed separately for each region. Sensitivity analysis: Samples will be stratified by collection time as described for the primary grading analysis, with RMSE, MAE, Bland-Altman limits and correlation coefficients computed separately for each time period. Sensitivity analysis: Agreement between AI and multiple pathologists will be assessed and compared using the same structure as the grading sensitivity analysis, with RMSE, MAE, Bland-Altman limits and correlation coefficients computed quantifying pairwise concordance. The analysis will be applied to the 349 cores annotated by all three pathologists (F.G., M.F., Os.A.) within the ProMort II software validation subsample. Secondary analysis: Cribriform cancer detection Cribriform pattern detection performance will be evaluated following the same analytical framework as the primary grading analysis, with the following modifications: All analyses will be conducted at core level, using reference standards aggregated from focus region annotations. Performance metrics: As a binary classification task, the prediction probability outputs will be converted to binary predictions using a pre-determined threshold of 0.5, which was optimised during model training through loss function weighting. Evaluation will include sensitivity, specificity, AUC-ROC, confusion matrices and calibration plot. Unweighted Cohen’s kappa will replace the weighted kappa measures for concordance assessment. Subgroup analysis: Geographical stratification will be conducted as described for the primary grading analysis, with sensitivity, specificity, confusion matrices and unweighted Cohen’s kappa computed separately for each region. Sensitivity analysis: Samples will be stratified by collection time as described for the primary grading analysis, with sensitivity, specificity, confusion matrices and unweighted Cohen’s kappa computed separately for each time period. Sensitivity analysis: Agreement between AI and multiple pathologists will be assessed and compared using the same structure as the grading sensitivity analysis, with unweighted kappa quantifying pairwise agreements. ProMort I: 288 cores annotated by all three pathologists (F.G., L.M., M.F.), and additional cores annotated by two pathologists: 193 cores by F.G. and L.M., and 266 cores by F.G. and M.F. ProMort II: 196 cores annotated by all three pathologists (F.G., M.F., Os.A.). Note that the AI system was developed to detect cribriform architecture regardless of its histological context—whether occurring within invasive acinar adenocarcinoma or intraductal carcinoma (IDC). This approach aligns with the 2019 ISUP consensus recommendations 7 , which advocate for combined assessment of invasive and intraductal cribriform patterns for prognostic evaluation and treatment planning. Secondary analysis: Perineural invasion detection Perineural invasion detection performance will be evaluated using an identical analytical framework as cribriform pattern detection. All analyses will be conducted at the core level with the same evaluation metrics (sensitivity, specificity, AUC-ROC, confusion matrices, calibration plot and unweighted Cohen’s kappa) using a 0.5 threshold for binary classification, subgroup analysis, and sensitivity analyses approaches as described for cribriform detection above. Analysis granularity considerations For Gleason score assessment, the primary reference pathologist F.G. reviewed all cores on all slides for each assigned patient according to the review protocols (Supplement 1,2,3). However, we report only core-level outcomes rather than patient-level analyses. Since patient-level reference annotations were not collected, deriving patient-level outcomes would require applying post-hoc aggregation rules (e.g., taking the maximum/average/majority Gleason score across cores) to both reviewer annotations and AI predictions. Such aggregation approaches can introduce systematic bias or mask model performance limitations, particularly given that clinical practice involves variable and subjective approaches to patient-level scoring 7 . Therefore, our concordance analyses focus on core-level AI-pathologist agreement as the most direct and unbiased assessment of model performance. For focus-region annotations for PNI and cribriform cancer detection, analyses are performed at the core level for several methodological reasons. Focus-region level comparison is not feasible due to inter-observer variability in spatial delineation, as pathologists may define different numbers and boundaries of focus regions within the same core based on varying interpretations of tissue gaps and morphological continuity. Slide-level and patient-level aggregations, while technically possible, would not provide meaningful analytical value beyond simple aggregation of core-level findings. Additionally, it is not uncommon for slides to contain only one core per slice, meaning core-level granularity could be used to approximate slide-level assessment. This allows the core-level validation results on ProMort to be compared with the AI model’s previous validation performance at slide level, providing important reference baselines for performance evaluation. Limitations and interpretive considerations Generalisability limitations The validation cohorts present spectrum and prevalence limitations that may affect generalisability. Both ProMort I and II utilise case-control designs with 50% cancer-related mortality, substantially higher than typical clinical populations, which may limit fair comparison of performance metrics with other AI validation studies using different cohort compositions. Additionally, ProMort I predominantly includes lower-grade cases (Gleason ≤7), creating spectrum bias that restricts assessment of AI performance across the full range of prostate cancer aggressiveness. These distributional characteristics should be considered when interpreting performance metrics or comparing results with other validation studies for AI-assisted cancer diagnostic systems. While subjects were excluded for various reasons, we specifically clarify two exclusion types that might be mistaken for sources of bias. Missing slide exclusions resulted from institutional slide retention policies (e.g., discarding slides older than 10 years) rather than diagnostic-related factors. Exclusions due to inadequate slide quality during review were minimal and did not systematically target specific patient subgroups or tumour characteristics. Information bias Although pathologists were blinded to original Gleason scores and individual case-control status, awareness that both ProMort I and II case-control subsamples have higher mortality rates compared to unselected clinical populations may introduce unconscious grading bias. Pathologists might unconsciously assign higher grades when reviewing cases from cohorts known to be enriched for fatal outcomes, which could artificially increase or decrease AI-pathologist agreement depending on whether the AI exhibits systematic tendencies for over- or undergrading, respectively. Inter-observer analysis limitations The ProMort I validation subsample employed a hierarchical annotation protocol where subsequent pathologist evaluations were contingent upon prior assessments. The second reviewer L.M. evaluated only cores classified as malignant by F.G., while M.F. assessed exclusively those cores exhibiting inter-observer discordance between F.G. and L.M. This design creates several methodological constraints that affect the interpretation of inter-observer analyses: Scope of analysis : Inter-observer comparisons are restricted to diagnostically challenging samples requiring adjudication rather than representing general diagnostic concordance across the full spectrum of cases. Underestimation of pathologist concordance : The exclusion of clear benign samples (where high agreement would be expected) leads to systematic underestimation of true pathologist-pathologist and pathologist-AI agreement compared to protocols evaluating complete sample sets, especially pronounced for F.G. vs. L.M. concordance. Comparative interpretation : Results should be interpreted as evaluating whether AI performance on diagnostically challenging samples is comparable to inter-pathologist agreement on the same difficult samples, rather than as measures of overall diagnostic capability. These constraints were predetermined by the original study design and cannot be modified retrospectively. ProMort II provides a more balanced inter-observer comparison framework for validation of these findings. Measurement bias For cribriform pattern detection, borderline cases were dichotomised as present/absent without an “equivocal” category. This forced dichotomisation may underestimate true AI-pathologist agreement, as disagreements on genuinely ambiguous cases are counted as errors rather than recognised as inherent uncertainty in the reference standard. ETHICS AND DISSEMINATION The study is conducted in agreement with the Declaration of Helsinki and approved by the Swedish Regional Ethics Review Board and the Swedish Ethical Review Authority (permits 2012/1586-31/1, 2016/613-31/2, 2019-01395, 2019-05220). Data were obtained from the National Prostate Cancer Register of Sweden (NPCR), which operates under the Swedish Patient Data Act (Patientdatalagen 2008:355, Chapter 7) and the EU General Data Protection Regulation (GDPR), without reliance on individual informed consent. The study results will be submitted for publication in an open-access format, regardless of whether the findings are positive, negative, or inconclusive in relation to the study hypothesis. AUTHOR CONTRIBUTIONS Study design: X.J., R.Z., Os.A., N.M., P.H.V., M.E., Ol.A., K.K. Data collection, curation and annotation: R.Z., Os.A., M.F., F.G., L.M., A.P. Data management and software: X.J., R.Z., N.M., S.E.B., K.S., L.X.L., A.P., P.H.V., Ol.A Drafting of the protocol: X.J., R.Z., Os.A., N.M., P.H.V., M.E., K.K. All authors have read and approved the final manuscript. K.K. is the guarantor. FUNDING P.H.V. received funding from ITEA-Vinnova (Symphony; 2022-01275). M.E. received funding from the Swedish Research Council, Swedish Cancer Society, Swedish Prostate Cancer Society, Nordic Cancer Union, Karolinska Institutet, and Region Stockholm. Ol.A. received funding from the Swedish Cancer Society (23 3256 S 01 H; 22 2324 Pj 01 H) and Radiumhemmets Forskningsfonder. K.K. received funding from the SciLifeLab & Wallenberg Data Driven Life Science Program (KAW 2024.0159), the David and Astrid Hägelen Foundation, Instrumentarium Science Foundation, KAUTE Foundation, Karolinska Institute Research Foundation, Orion Research Foundation and Oskar Huttunen Foundation. DATA AVAILABILITY A subset of the data used for model training (STHLM3 and RUMC cohorts) are available for non-commercial purposes subject to a CC BY-SA-NC 4.0 license as part of the PANDA challenge dataset and are freely downloadable after registration at https://www.kaggle.com/c/prostate-cancer-grade-assessment . The ProMort validation data underlying this article cannot be shared publicly due to privacy concerns. Data from these cohorts may be made available through contact with P.H.V. (Karolinska University Hospital) under appropriate research collaboration and data-sharing agreements. COMPETING INTERESTS N.M., K.K. and M.E. are co-founders and shareholders of Clinsight AB. All other authors have no competing interests to declare. The funder (Karolinska Institutet) did not influence the results/outcomes of the study despite author affiliations with the funder. PATIENT AND PUBLIC INVOLVEMENT We maintained ongoing dialogue with patient organisations, healthcare providers, public authorities, and other key stakeholders throughout the research process. Insights gained from these interactions directly shaped both the design of our study and the development of our AI algorithms. Moving forward, we will collaborate with patient organisations in disseminating the study results to ensure the findings are accessible and valuable to the broader community. STUDY STATUS The key timepoints for this retrospective AI validation study are: 1) Confirmation of all AI model updates and acquisition of the final AI model version, 2) Establishment of the pre-specified statistical analysis plan for validation data, 3) Conducting the final evaluation on validation data according to the pre-specified plan. Respecting this timeline ensures no information leakage from the validation data influences the AI model design, and conversely, that validation analysis plans are not biased by prior knowledge of the latest model’s performance or limitations. The study status on this timeline is as follows: 31 July 2025: All model updates were confirmed and the latest AI model version was obtained and locked for validation purposes. 22 September 2025: The protocol covering AI model evaluation on PROMORT validation datasets was submitted to be made publicly available as a pre-print on medRxiv. October 2025: Final evaluation of the AI model on the PROMORT validation datasets for cancer detection and Gleason grading will be conducted according to the pre-specified analysis plan with results to be published in a peer-reviewed journal. ACKNOWLEDGEMENTS Computing resources are provided by the National Academic Infrastructure for Supercomputing in Sweden (NAISS) and the Swedish National Infrastructure for Computing (SNIC) at C3SE partially funded by the Swedish Research Council through grant agreement no. 2022-06725 and no. 2018-05973, and by the supercomputing resource Berzelius provided by the National Supercomputer Centre at Linköping University and the Knut and Alice Wallenberg Foundation. REFERENCES 1. ↵ Gleason , D. F. & Mellinger , G. T . Prediction of prognosis for prostatic adenocarcinoma by combined histological grading and clinical staging . J Urol 111 , 58 – 64 ( 1974 ). OpenUrl CrossRef PubMed Web of Science 2. ↵ Phillips , J. L. & Sinha , A. A . Patterns, art, and context: Donald Floyd Gleason and the development of the Gleason grading system . Urology 74 , 497 – 503 ( 2009 ). OpenUrl CrossRef PubMed 3. ↵ Ozkan , T. A. et al. Interobserver variability in Gleason histological grading of prostate cancer . Scand J Urol 50 , 420 – 424 ( 2016 ). OpenUrl PubMed 4. ↵ Egevad , L. et al. Standardization of Gleason grading among 337 European pathologists . Histopathology 62 , 247 – 256 ( 2013 ). OpenUrl PubMed 5. ↵ Epstein , J. I. , Allsbrook , W. C. , Jr . , Amin , M. B. , Egevad , L. L. & ISUP Grading Committee . The 2005 International Society of Urological Pathology (ISUP) Consensus Conference on Gleason Grading of Prostatic Carcinoma . Am J Surg Pathol 29 , 1228 – 1242 ( 2005 ). OpenUrl CrossRef PubMed Web of Science 6. ↵ Epstein , J. I. et al. The 2014 International Society of Urological Pathology (ISUP) Consensus Conference on Gleason Grading of Prostatic Carcinoma: Definition of Grading Patterns and Proposal for a New Grading System . Am J Surg Pathol 40 , 244 – 252 ( 2016 ). OpenUrl CrossRef PubMed 7. ↵ van Leenders , G. J. L. H. , et al. The 2019 International Society of Urological Pathology (ISUP) Consensus Conference on Grading of Prostatic Carcinoma . Am J Surg Pathol 44 , e87 – e99 ( 2020 ). OpenUrl CrossRef PubMed 8. ↵ Egevad , L. et al. Interobserver reproducibility of cribriform cancer in prostate needle biopsies and validation of International Society of Urological Pathology criteria . Histopathology 82 , 837 – 845 ( 2023 ). OpenUrl PubMed 9. ↵ Egevad , L. et al. Interobserver reproducibility of perineural invasion of prostatic adenocarcinoma in needle biopsies . Virchows Arch 478 , 1109 – 1116 ( 2021 ). OpenUrl PubMed 10. ↵ Patel , V. , Hubbard , S. & Huang , W . Comparison of two commonly used methods in measurement of cancer volume in prostate biopsy . International Journal of Clinical and Experimental Pathology 13 , 664 ( 2020 ). 11. ↵ Bernhardt , M. et al. Unexpectedly high variability in determining tumour extent in prostatic biopsies: implications for active surveillance . Histopathology 86 , 627 – 639 ( 2025 ). OpenUrl PubMed 12. ↵ Campanella , G. et al. Clinical-grade computational pathology using weakly supervised deep learning on whole slide images . Nat Med 25 , 1301 – 1309 ( 2019 ). OpenUrl CrossRef PubMed 13. ↵ Ström , P. et al. Artificial intelligence for diagnosis and grading of prostate cancer in biopsies: a population-based, diagnostic study . Lancet Oncol 21 , 222 – 232 ( 2020 ). OpenUrl CrossRef PubMed 14. ↵ Bulten , W. et al. Automated deep-learning system for Gleason grading of prostate cancer using biopsies: a diagnostic study . Lancet Oncol 21 , 233 – 241 ( 2020 ). OpenUrl CrossRef PubMed 15. ↵ Bulten , W. et al. Artificial intelligence for diagnosis and Gleason grading of prostate cancer: the PANDA challenge . Nat Med 28 , 154 – 163 ( 2022 ). OpenUrl CrossRef PubMed 16. ↵ Raciti , P. et al. Clinical Validation of Artificial Intelligence-Augmented Pathology Diagnosis Demonstrates Significant Gains in Diagnostic Accuracy in Prostate Cancer Detection . Arch Pathol Lab Med 147 , 1178 – 1185 ( 2023 ). OpenUrl PubMed 17. ↵ Chen , R. J. et al. Towards a general-purpose foundation model for computational pathology . Nat Med 30 , 850 – 862 ( 2024 ). OpenUrl CrossRef PubMed 18. ↵ Vorontsov , E. et al. A foundation model for clinical-grade computational pathology and rare cancers detection . Nat Med 30 , 2924 – 2935 ( 2024 ). OpenUrl CrossRef PubMed 19. ↵ Zimmermann , E. , et al. Virchow2: Scaling self-supervised mixed magnification models in pathology . arXiv [cs.CV] ( 2024 ) doi: 10.48550/ARXIV.2408.00738 . OpenUrl CrossRef 20. ↵ Lu , M. Y. et al. A visual-language foundation model for computational pathology . Nat Med 30 , 863 – 874 ( 2024 ). OpenUrl CrossRef PubMed 21. ↵ Matthews , G. A. , McGenity , C. , Bansal , D. & Treanor , D . Public evidence on AI products for digital pathology . NPJ Digit Med 7 , 300 ( 2024 ). 22. ↵ McGenity , C. et al. Artificial intelligence in digital pathology: a systematic review and meta-analysis of diagnostic test accuracy . NPJ Digit Med 7 , 114 ( 2024 ). 23. ↵ Ochi , M. et al. Registered multi-device/staining histology image dataset for domain-agnostic machine learning models . Sci Data 11 , 330 ( 2024 ). 24. ↵ Ji , X. et al. Physical Color Calibration of Digital Pathology Scanners for Robust Artificial Intelligence-Assisted Cancer Diagnosis . Mod Pathol 38 , 100715 ( 2025 ). 25. ↵ Zelic , R. et al. Interchangeability of light and virtual microscopy for histopathological evaluation of prostate cancer . Sci Rep 11 , 3257 ( 2021 ). OpenUrl PubMed 26. ↵ Mun , Y. , Paik , I. , Shin , S.-J. , Kwak , T.-Y. & Chang , H . Yet Another Automated Gleason Grading System (YAAGGS) by weakly supervised deep learning . NPJ Digit Med 4 , 99 ( 2021 ). 27. ↵ Guan , H. , Bates , D. & Zhou , L. Keeping medical AI healthy: A review of detection and correction methods for system degradation . arXiv [cs.AI] ( 2025 ) doi: 10.48550/ARXIV.2506.17442 . OpenUrl CrossRef 28. ↵ McGenity , C. , Bossuyt , P. & Treanor , D . Reporting of Artificial Intelligence Diagnostic Accuracy Studies in Pathology Abstracts: Compliance with STARD for Abstracts Guidelines . J Pathol Inform 13 , 100091 ( 2022 ). 29. ↵ Mulliqi , N. , et al. Foundation models -- A panacea for artificial intelligence in pathology? arXiv [cs.CV] ( 2025 ) doi: 10.48550/ARXIV.2502.21264 . OpenUrl CrossRef 30. ↵ Mulliqi , N. et al. Development and retrospective validation of an artificial intelligence system for diagnostic assessment of prostate biopsies: study protocol . BMJ Open 15 , e097591 ( 2025 ). OpenUrl Abstract / FREE Full Text 31. ↵ Zelic , R. et al. Estimation of Relative and Absolute Risks in a Competing-Risks Setting Using a Nested Case-Control Study Design: Example From the ProMort Study . Am J Epidemiol 188 , 1165 – 1173 ( 2019 ). OpenUrl PubMed 32. ↵ Zelic , R. et al. Prognostic Utility of the Gleason Grading System Revisions and Histopathological Factors Beyond Gleason Grade . Clin Epidemiol 14 , 59 – 70 ( 2022 ). OpenUrl 33. ↵ Tejani , A. S. , et al. Checklist for Artificial Intelligence in Medical Imaging (CLAIM): 2024 Update . Radiol Artif Intell 6 , e240300 ( 2024 ). OpenUrl 34. ↵ Sounderajah , V. et al. The STARD-AI reporting guideline for diagnostic accuracy studies using artificial intelligence . Nat Med ( 2025 ) doi: 10.1038/s41591-025-03953-8 . OpenUrl CrossRef 35. ↵ Cohen , J. F. et al. STARD 2015 guidelines for reporting diagnostic accuracy studies: explanation and elaboration . BMJ Open 6 , e012799 ( 2016 ). OpenUrl Abstract / FREE Full Text 36. ↵ Lianas , L. et al. An open-source platform for structured annotation and computational workflows in digital pathology research . Sci Rep 15 , 28910 ( 2025 ). 37. ↵ Van Calster , B. , Van Belle , V. , Vergouwe , Y. & Steyerberg , E. W . Discrimination ability of prediction models for ordinal outcomes: relationships between existing measures and a new measure . Biom J 54 , 674 – 685 ( 2012 ). OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted September 23, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol Xiaoyi Ji , Renata Zelic , Oskar Aspegren , Nita Mulliqi , Michelangelo Fiorentino , Francesca Giunchi , Luca Molinaro , Sol Erika Boman , Kelvin Szolnoky , Luana Xuan Liu , Andreas Pettersson , Per Henrik Vincent , Martin Eklund , Olof Akre , Kimmo Kartasalo medRxiv 2025.09.22.25336169; doi: https://doi.org/10.1101/2025.09.22.25336169 Share This Article: Copy Citation Tools Retrospective Validation of an Artificial Intelligence System for Diagnostic Assessment of Prostate Biopsies on the ProMort Cohort: Study Protocol Xiaoyi Ji , Renata Zelic , Oskar Aspegren , Nita Mulliqi , Michelangelo Fiorentino , Francesca Giunchi , Luca Molinaro , Sol Erika Boman , Kelvin Szolnoky , Luana Xuan Liu , Andreas Pettersson , Per Henrik Vincent , Martin Eklund , Olof Akre , Kimmo Kartasalo medRxiv 2025.09.22.25336169; doi: https://doi.org/10.1101/2025.09.22.25336169 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Pathology Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4425) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15221) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6588) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9219) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (710) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffabc9eff330db4',t:'MTc3OTQ0MTU0MA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();