Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study

Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study View ORCID Profile Lishan Cai , Mohamed A. Abdelatty , Luyi Han , Doenja M. J. Lambregts , Joost van Griethuysen , View ORCID Profile Eduardo Pooch , Regina G.H. Beets-Tan , Sean Benson , View ORCID Profile Joren Brunekreef , Jonas Teuwen doi: https://doi.org/10.1101/2024.10.15.24315517 Lishan Cai a Department of Radiology, Netherlands Cancer Institute , The Netherlands b GROW School for Oncology and Developmental Biology, Maastricht University Medical Centre , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Lishan Cai Mohamed A. Abdelatty a Department of Radiology, Netherlands Cancer Institute , The Netherlands c Department of Diagnostic and Interventional Radiology, Kasr Al-Ainy Hospital , Egypt Find this author on Google Scholar Find this author on PubMed Search for this author on this site Luyi Han a Department of Radiology, Netherlands Cancer Institute , The Netherlands d Department of Radiology and Nuclear Medicine, Radboud University Medical Centre , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Doenja M. J. Lambregts a Department of Radiology, Netherlands Cancer Institute , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Joost van Griethuysen a Department of Radiology, Netherlands Cancer Institute , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eduardo Pooch a Department of Radiology, Netherlands Cancer Institute , The Netherlands b GROW School for Oncology and Developmental Biology, Maastricht University Medical Centre , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Eduardo Pooch Regina G.H. Beets-Tan a Department of Radiology, Netherlands Cancer Institute , The Netherlands b GROW School for Oncology and Developmental Biology, Maastricht University Medical Centre , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sean Benson a Department of Radiology, Netherlands Cancer Institute , The Netherlands e Department of Cardiology, Amsterdam Cardiovascular Sciences, Amsterdam University Medical , Centre, The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site Joren Brunekreef f Department of Radiation Oncology, Netherlands Cancer Institute , The Netherlands h University of Amsterdam, Faculty of Science , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Joren Brunekreef Jonas Teuwen f Department of Radiation Oncology, Netherlands Cancer Institute , The Netherlands g Radboud University Medical Center, Department of Medical Imaging , The Netherlands h University of Amsterdam, Faculty of Science , The Netherlands Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: j.teuwen{at}nki.nl Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Accurate rectal tumor segmentation using magnetic resonance imaging (MRI) is paramount for effective treatment planning. It allows for volumetric and other quantitative tumor assessments, potentially aiding in prognostication and treatment response evaluation. Manual delineation of rectal tumors and surrounding structures is time-consuming and typically. Over the past few years, deep learning has shown strong results in automated tumor segmentation in MRI. Current studies on automated rectal tumor segmentation, however, focus solely on tumoral regions without considering the rectal anatomical entities and often lack a solid multicenter external validation. In this study, we improved rectal tumor segmentation by incorporating anomaly maps derived from anatomical inpainting. This inpainting was implemented using a U-Net-based model trained to reconstruct a healthy rectum and mesorectum from prostate T2-weighted images (T2WI). The rectal anomaly maps were generated from the difference between the original rectal and reconstructed pseudo-healthy slices during inference. The derived anomaly maps were used in the downstream tumor segmentation tasks by fusing them as an additional input channel (AAnnUNet). Alternative methods for integrating rectal anatomical knowledge were evaluated as baselines, including Multi-Target nnUNet (MTnnUNet), which added rectum and mesorectum segmentation as auxiliary tasks, and Multi-Channel nnUNet (MCnnUNet), which utilized rectum and mesorectum masks as an additional input channel. As part of this study, we benchmarked nine models for rectal tumor segmentation on a large multicenter dataset of preoperative T2WI as the baseline and nnUNet outperformed the other eight models on the external dataset. The MTnnUNet demonstrated improvements in both supervised and semi-supervised settings (AI-generated rectum and mesoretum were used) compared to nnUNet, while the MCnnUNet showed benefits only in the semi-supervised setting. Importantly, anomaly maps were strongly associated with tumoral regions, and their integration within AAnnUNet led to the best tumor segmentation results across both settings. The effectiveness of AAnnUNet demonstrated the value of the anomaly maps, indicating a promising direction for improving rectal tumor segmentation and model robustness for multicenter data. 1. Introduction Magnetic Resonance Imaging (MRI) plays a pivotal role in staging rectal cancer and selecting treatment plans, providing valuable information regarding the extent of tumor infiltration within and beyond the bowel wall, and into critical anatomical structures, including perirectal vessels, the mesorectal fascia (MRF), peritoneum, and neighboring pelvic organs ( Beets-Tan et al., 2018 ; MERCURY Study Group, 2007 ). T2-weighted imaging (T2WI) forms the mainstay of the MRI protocol because of its superior soft tissue contrast to discern the different layers of the rectal wall, mesorectal fat, adjacent vessels, and MRF to allow for detailed local staging ( Horvat et al., 2019 ; Suzuki et al., 2008 ). Precisely segmenting the tumor is an important task in rectal cancer management. Tumor segmentations are utilized for several purposes including radiation treatment planning, volumetric analysis, and extraction of imaging biomarkers, which may serve as a basis for prognostication and treatment response evaluation. Manual rectal tumor delineation by experienced radiologists is considered the current gold standard. Nevertheless, it is time-consuming and subject to substantial intra- and inter-observer variation ( Hearn et al., 2020 ; Irving et al., 2016 ; Trebeschi et al., 2017 ). Developing an accurate, generalizable, and robust rectal tumor segmentation model can help reduce this variability and assist in standardizing several steps of diagnostic and therapeutic rectal cancer management. Deep learning (DL) has seen a rapid uptake in several fields, achieving state-of-the-art results in multiple medical image analysis tasks ( Razzak et al., 2018 ). In particular, Convolutional Neural Network-based (CNN) based DL approaches excel in learning image representations from annotated data by using learnable feature extraction filters and sequential convolution, activation, and pooling operations. Among CNN architectures, the U-Net ( Ronneberger et al., 2015 ) and its variants are the most popular architecture in medical image segmentation. Studies ( Jian et al., 2018 ; Knuth et al., 2022 ; Wang et al., 2018 ) have explored the ability of U-Net and other 2D CNNs on rectal tumor segmentation based on T2WI. These CNNs achieved a Dice similarity coefficient score (DSC) ranging from 0.59 to 0.84 for tumor segmentation. Although 2D CNNs are less computationally expensive, 3D models can leverage richer context to improve predictions ( Mlynarski et al., 2019 ). ( Hamabe et al., 2022 ) implemented a 3D U-Net, achieving an averaged DSC of 0.73 (0.60-0.80) over 10-fold cross-validation for rectal tumor segmentation. However, no external validation was done in their study. Besides CNN-based models, transformer-based architectures are also being applied in medical image analysis due to their ability to access long-range semantic information ( Xiao et al., 2023 ). ( Li et al., 2023 ) proposed RTAU-Net, a novel 3D dual path fusion network containing a transformer encoder for extracting global contour information of the tumor. RTAU-Net achieved an averaged DSC of 0.80 and 0.68 in data from two respective medical centers. However, RTAU-Net requires the manual removal of tumor-free slices, which hinders the fully automated implementation of the model. Additionally, RTAU-Net was not compared with the state-of-the-art medical segmentation networks, such as nnUNet ( Isensee et al., 2021 ), a self-configuring implementation of the U-Net architecture, or nnFormer ( Zhou et al., 2023 ), which introduces 3D transformer blocks on top of nnUNet. Besides rectal tumor delineation, some studies also demonstrated that CNNs can accurately delineate anatomical structures such as rectum and mesorectum or perirectal fat with DSC above 0.90 ( DeSilvio et al., 2023 ; Hamabe et al., 2022 ; Kim et al., 2019 ). Automated rectum and mesorectum delineation could potentially improve radiological evaluation. The prognosis for rectal cancer depends on how far the tumor has infiltrated the layers of the rectal wall and the mesorectum, and the successful attainment of negative circumferential resection margins (CRMs) through surgical intervention ( Nagtegaal et al., 2004 ). Additionally, Lee et al. ( Lee et al., 2019 ) demonstrated that a 2D model’s variance in tumor regions can be reduced by 90% by incorporating rectal segmentation on the model’s objective. Integrating rectal anatomical knowledge can provide a more comprehensive representation of the T2WI, allowing the model to learn richer and more nuanced patterns, leading to better performance on unseen data. However, the impact of adding rectal anatomical structures including mesorectum for rectal tumor segmentation has not been investigated in a multi-institutional setting. Also, incorporating rectal anatomical structure has so far been limited to adding additional segmentation tasks as presented in ( Lee et al., 2019 ). Unlike for medical challenges with public datasets like brain tumor segmentation ( Menze et al., 2015 ) or clinically significant prostate lesion segmentation (PICAI) ( Saha et al., 2023 ), there is no large multicenter publicly available MRI dataset for rectal cancer studies. This makes it difficult to benchmark different models. An extensive external validation study with multicenter data is highly desirable to compare different deep learning segmentation approaches. In this article, we had the following contributions: We developed and evaluated a rectal tumor segmentation model incorporating anomaly maps from anatomical inpainting, showing improved performance. To generate the anomaly maps, we proposed and evaluated a novel end-to-end rectal anatomical inpainting model, trained exclusively on prostate T2WI, to detect and highlight anomalous areas in rectal T2WI. The model trained on prostate T2WI was applied to rectal T2WI, challenging traditional domain-specific practices and demonstrating the potential for transfer learning. The derived rectal anomaly maps can also be integrated into other clinical downstream tasks. As part of our study, we benchmarked nine 3D deep learning models for rectal tumor segmentation on a large multicenter dataset of T2WI. We developed and evaluated a 3D deep learning model specifically to segment rectal anatomical structures, including rectum and mesorectum. We explored different strategies for incorporating rectal anatomical information into rectal tumor segmentation, including the integration of anomaly maps derived from the inpainting model, adding rectal structures as additional tasks, and utilizing them as prior knowledge. We released the rectum and mesorectum masks of 100 prostate T2WIs from PICAI dataset, annotated by radiologist. We uploaded the model weights for rectum and mesorectum segmentation, as well as the weights for MTnnUNet, MCnnUNet, and AAnnUNet. 2 Related Work 2.1 Reconstruction-based Medical Anomaly Detection Supervised learning requires a substantial amount of reliably labeled data, which is often difficult to collect in medical imaging. Therefore, methods requiring partly labeled data (semi-supervised) or no labeling (unsupervised methods) have attracted increased attention. Anomaly detection is a method that can use semi-supervised or unsupervised techniques to address medical imaging tasks such as segmentation. Generative models are frequently used in the field of anomaly detection due to their effectiveness. The mode is often trained to reconstruct images from a specific data distribution (e.g., normal tissue). These models, when confronted with images outside of this distribution (such as those containing tumors), often struggle to accurately reconstruct the anomalous regions, resulting in higher reconstruction errors in those areas Generative adversarial networks (GANs) ( Goodfellow et al., 2014 ), Auto-Encoders (AE), and their variants including Variational AE (VAE) ( Kingma and Welling, 2022 ) and Vector Quantized VAE (VQ-VAE) ( van den Oord et al., 2017 ) were proven to be promising ( Baur et al., 2021 ; Chen et al., 2020 ; Pinaya et al., 2022 ) in this field. However, these methods have presented several challenges. They struggle to learn and represent healthy anatomical structures when using full image resolution. Additionally, when reconstructing the entire image, the highlighted regions might not exclusively correspond to diseased areas. As a result, the final anomaly maps may become more sensitive to other artifacts. Finally, if it is in the self-supervised setting where healthy or normal samples are not available for training, these methods can sometimes can successfully reconstruct anomalies due to a high generalization capacity ( Gong et al., 2019 ). 2.2.1 Partial Image Reconstruction via Inpainting for Medical Anomaly Detection Instead of image-to-image reconstruction methods, inpainting focuses on filling in missing or occluded parts of an image by leveraging the surrounding context. The masks used for inpainting are generally independent of the dataset. ( Nguyen et al., 2021 ) implemented an inpainting-based brain tumor segmentation pipeline for T1-weighted MRI, where anomalous regions were determined by identifying areas of highest reconstruction loss. With prior anatomical knowledge, inpainting can focus solely on high-risk regions, reducing the impact from the background. ( Yeganeh et al., 2022 ) developed an anatomy-aware masking strategy for inpainting to effectively aid the model in learning the shape representation of the organs of interest. ( Woo et al., 2024 ) have proposed a UNet-based model for detecting bone lesions in knee MR images through reconstruction via inpainting and demonstrated that detected anomalies can be further utilized for segmentation. The core idea of anomaly detection through anatomical inpainting involves masking the region of interest (ROI), typically covering the relevant anatomical structures. The inpainting model is trained to fill in the masked area without potential anomalies. The discrepancy between the original and reconstructed images is then utilized to identify anomalies. In this study, the anatomical structures strongly associated with rectal cancer, including the rectum and mesorectum (fatty tissue surrounding the rectum), were masked, and the inpainting model was trained using prostate T2WI images from the PICAI dataset ( Saha et al., 2023 ). Despite the different fields of view between prostate T2WI and rectal T2WI, they overlap in anatomical structures. Most prostate T2WI contains a healthy rectum and mesorectum, ensuring the inpainting model was trained to learn the distribution of the healthy tissues. The inferred reconstructed rectal T2WI slices can then be used to generate anomaly maps, highlighting potentially tumoral areas. 3. Methods 3.1 Rectal Anomaly Detection with Masked T2WI using Anatomical Inpainting The overall pipeline, inspired by ( Han et al., 2024 , 2023 ), for detecting rectal anomalies is shown in Figure 2 . Prostate T2WI with masked rectum and mesorectum were used to train the model for their reconstruction. The inpainting model was adapted from ( Han et al., 2024 ), an end-to-end MRI sequence generation framework. The framework consists of two stages. In the first stage, only the reconstruction loss is optimized for the encoder and generator. In the second stage, both adversarial loss and cycle-consistent loss are incorporated in addition to the reconstruction loss, with the optimization applied to the encoder, generator, and discriminator. The training of the anatomical inpainting was based on 2D slices. The inpainting model contains an encoder E and a decoder G . A masked (rectum and mesorectum) 2D T2WI slice X , can be compressed by E into a latent space z = E ( X ) and G can reconstruct the original slice from the latent representation z . The skip connections were added to recover fine-grained details. To enforce similarity between the generated slices and the actual slices, a supervised reconstruction loss is used: where X is the original slice, and X ′ = G ( E ( X )) is the restored image. ‖⋅‖ 1 is the L 1 loss and L p is the perceptual loss from pre-trained VGG19, which involves comparing high-level features (not just pixel values) from both the generated and reference images ( Johnson et al., 2016 ). Instead of measuring raw pixel differences, it evaluates how similar the images are in terms of their content and style, based on features extracted from different layers of the VGG19 network. λ r and λ p are weight factors, for which the respective values 10 and 0.01 were chosen empirically. Download figure Open in new tab Figure 1: The rectal tumor segmentation performances of various 3D DL models were compared. All the models were trained with training cohort 1 including 39 samples from Center 9 only and externally tested with 666 samples from 9 centers. Download figure Open in new tab Figure 2: The anatomical inpainting workflow. The inpainting model containing an encoder E and a decoder G was trained using prostate T2WI with a healthy rectum and mesorectum. The trained model was inferred across the rectal dataset to generate a reconstructed pseudo healthy rectum and mesorectum. The difference between the reconstructed and the original image is the anomaly map where tumoral regions were highlighted. For the second stage of the training, the adversarial loss and cycle-consistent loss ( Zhu et al., 2017 ) were added on top of the reconstruction loss to ensure that the inpainted images were both realistic and consistent with the original images. The adversarial loss helps to ensure that the completed regions look realistic and blend seamlessly with the surrounding areas and the cycle-consistent loss focuses on preserving the original structure by ensuring the inpainted image can be accurately reconstructed back to the original. where X ′′ = G ( E ( X ′ )) and ‖⋅‖ 2 is the L 2 loss and D is the discriminator. The anomaly maps are then defined as the absolute differences between the reconstructed slice and the original slice, Let I be an image with intensity values. The normalization scheme involved the following steps: First Compute the 0.5th percentile ( l ) and the 99.5th percentile (ℎ) of the intensity values in the image and normalize the intensity values in the image using the computed percentiles. The rectum was masked with a value of 0, while the mesorectum was masked with a value of 0.5. The model input patch size was (384, 384), with a batch size of 1. We used the AdamW optimizer to train the network (both stages) with β 0.9 and 0.95, an initial learning rate of 0.0001, a weight decay factor of 0.05, and following a polynomial decay. The model was implemented in PyTorch (Torch version 2.1.2) and the training was conducted on an NVIDIA RTX A6000 GPU. 3.2 Study Design To train the inpainting model, 100 prostate T2WI with manually segmented rectum and mesorectum masks were split into 80 for training and 20 for internal validation. The model was additionally tested on 200 prostate T2WIs and the entire rectal dataset, comprising 705 T2WI. However, because annotating rectal structures is labor-intensive, only 180 rectal randomly selected samples have radiologist-annotated masks for the rectum and mesorectum. To address this, a nnUNet, defined as anatomy nnUNet, was trained specifically to segment the rectum and mesorectum using a dataset of 39 samples (training cohort 1) from a single center, which had the highest number of manually annotated rectum and mesorectum masks among nine centers. Some studies have demonstrated that CNNs can accurately delineate anatomical structures such as rectum and mesorectum or perirectal fat with DSC above 0.90 ( DeSilvio et al., 2023 ; Hamabe et al., 2022 ; Kim et al., 2019 ). The model was then evaluated on 141 external samples, see Figure 3 . This nnUNet was then used to infer all the rectum and mesorectum masks across the whole rectal dataset. The predicted rectum and mesorectum masks were defined as AI-generated pseudo rectum and mesorectum masks. Download figure Open in new tab Figure 3: ( a) Model definition. A: a nnUNet for tumor segmentation only; B: MTnnUNet: multi-target nnUNet, segmenting tumor, rectum, and mesorectum simultaneously, C: MCnnUNet: multi-channel nnUNet with rectum and mesorectum masks together with T2WI as input, D: AAnnUNet: Anomaly-Aware nnUNet, with anomaly maps derived from anatomical inpainting as additional input E: Anatomy nnUNet, to segment rectum and mesorectum. F: MTnnUNet but trained with AI-generated rectum and mesorectum, G: MCnnUNet but using AI-generated rectum and mesorectum, H: AAnnUNet but using AI-generated rectum and mesorectum (b) The training scheme for different models. 5F-CV: 5-fold cross-validation. Models A (A-H) are defined in ( a ). We incorporated anomaly maps generated by the inpainting model into downstream rectal tumor segmentation tasks by adding them as an additional input for nnUNet, called Anomaly-Aware nnUNet (AAnnUNet). We compared this approach with other strategies for integrating anatomical knowledge into tumor segmentation, including Multi-Target nnUNet (MTnnUNet) and Multi-Channel nnUNet (MCnnUNet). MTnnUNet added rectum and mesorectum segmentation as auxiliary tasks, and Multi-Channel nnUNet (MCnnUNet) utilized rectum and mesorectum masks as additional input channels. First, we established a baseline for rectal tumor segmentation by comparing the performance of nine 3D deep learning models, see Figure 1 , including UNet ( Çiçek et al., 2016 ), ResUNet ( Diakogiannis et al., 2020 ), UNetR ( Hatamizadeh et al., 2022b ), SwinUNetR ( Hatamizadeh et al., 2022a ), AttentionUNet (Atten-UNet) ( Oktay et al., 2018 ), MedFormer ( Gao et al., 2023 ), nnFormer ( Zhou et al., 2023 ), U-Mamba (bot) ( Ma et al., 2024 ) and nnUNet ( Isensee et al., 2021 ). All the models underwent training using training cohort 1 with a 5-fold cross-validation. Subsequently, models were externally tested on the remaining 666 samples from nine centers. MTnnUNet, MCnnUNet, and AAnnUNet were trained on training cohort 1 using 5-fold cross-validation and tested on 666 samples from 9 centers. For the inference of MCnnUNet and AAnnUNet, AI-generated pseudo rectum and mesorectum were used. With AI-generated pseudo anatomical structures, MTnnUNet, MCnnUNet, and AAnnUNet were also trained on a larger cohort comprising 141 samples (training cohort 2) using 5-fold cross-validation and tested on 564 samples from eight centers. Instead of relying on ground truth rectum and mesorectum masks, AI-generated annotations were employed in training. This method merged AI-labeled anatomical structures with manually labeled tumors, indicating a semi-supervised learning approach. 3.3 Segmentation Models UNet ( Çiçek et al., 2016 ), extends the previous u-net architecture from Ronneberger et al., 2015 ( Ronneberger et al., 2015 ) by replacing all 2D operations with their 3D counterparts. ResUNet ( Diakogiannis et al., 2020 ), is a modified version of UNet. It replaces double convolution layers of UNet with residual blocks from ResNet ( He et al., 2016 ), incorporating shortcut connections for faster convergence. This adaptation works in both 2D and 3D settings, enhancing performance in capturing complex patterns. UNETR ( Hatamizadeh et al., 2022b ), adopts a ViT-inspired encoder and employs a CNN decoder for 3D image segmentation. The images are initially divided into patches, which are linearly transformed into token embeddings. These tokens undergo processing through a self-attention block, akin to ViT. To manage the quadratic complexity of self-attention, the patch size is set to be relatively large (16) to prevent overly long sequence lengths. SwinUNETR ( Hatamizadeh et al., 2022a ), reformulates the segmentation task as a sequence-to-sequence prediction using a Swin Transformer as the encoder. The encoder is then connected to a Fully Convolutional Neural Network (FCNN)-based decoder through skip connections. Attention UNet ( Oktay et al., 2018 ), introduces an attention-gating module to UNet to enhance its ability to suppress irrelevant regions and highlight salient features crucial for a given task. nnFormer ( Zhou et al., 2023 ), is a 3D transformer for volumetric medical image segmentation nnFormer combines interleaved convolution and self-attention operations. It introduces a special self-attention mechanism to understand both local and global aspects of the image volume. To improve efficiency, it also uses skip attention instead of the usual concatenation or summation operations. MedFormer ( Gao et al., 2023 ), is a transformer-based designed to handle scalable 3D medical image segmentation, including three crucial components: a beneficial inductive bias, hierarchical modeling using linear-complexity attention, and multi-scale feature fusion that combines spatial and semantic information globally. MedFormer can learn from both small and large scale data without pre-training. U-Mamba ( Ma et al., 2024 ), is inspired by the State Space Sequence Models (SSMs) ( Gu et al., 2021 ), which are known for their ability to handle long sequences. The model is designed specifically for biomedical image segmentation with the hybrid CNN-SSM block that integrates the local feature extraction power of convolutional layers with the abilities of SSMs to capture the long-range dependency. nnUNet ( Isensee et al., 2021 ), is a self-configuring framework for medical image segmentation. It utilizes UNet as its architecture but offers a specialized preprocessing, training technique, and hyper-parameter configuration. nnUNet achieves state-of-the-art performance on several medical image segmentation challenges with a relatively simple architectural design. Anatomy nnUNet , is the nnUNet trained to segment rectal-related anatomical structures including the rectum and mesorectum. MTnnUNet , is the nnUNet trained to segment rectum, mesorectum, and rectal tumors. MCnnUNet , is the nnUNet trained to segment rectal tumors with rectum and mesorectum masks as additional input channels. AAnnUNet , is the nnUNet trained to segment rectal tumors with anomaly maps M derived from anatomical inpainting. For rectal tumor segmentation, the imaging preprocessing approach was adopted from nnUNet, which included ZScoreNormalization for standardizing intensities, uniform resampling of all images, and a cropping process. All the segmentation models were implemented in PyTorch (Torch version 2.1.2) and trained on an NVIDIA A6000 GPU with randomly initialized weights without transfer learning. The batch size was set to 2 and the models were trained for 1000 epochs with the SGD optimizer. The loss function is the sum of the cross-entropy and Dice loss. During inference, predictions were obtained by averaging the outputs of each model resulting from the 5-fold cross-validation procedure. 3.4 Evaluation Metrics and Statistical Analysis Statistical analysis was conducted in Python (version 3.9) with the SciPy package (version 1.13.1). To measure the performance of image reconstruction, Structural Similarity Index Measurement (SSIM), Peak Signal-to-NoiseRatio (PSNR) were used. To measure the segmentation performance, the Dice Similarity Coefficient score (DSC) and 95% Hausdorff Distance (HD) were utilized on both cross-validation and external tests. The characteristic differences of cohorts were compared by the Kruskal-Wallis test. The Mann–Whitney U-test was used to compare the difference of indicators among different methods. The model performance differences were calculated using the paired sample t-test. All statistical analyses were two-sided and p-values below 0.05 were regarded as statistically significant. 95% confidence intervals were generated using the bootstrap method with 10,000 replications. 4. Results 4.1 Dataset and Patient Characteristics As a part of a previous institutional review board approved multicenter study project ( Bogveradze et al., 2022 ; Cai et al., 2024 ; El Khababi et al., 2023 ; “ESGAR 2020 Book of Abstracts,” 2020 ; Schurink et al., 2023 , 2022 ) the clinical and imaging data from 1426 patients with biopsy-proven rectal cancer were retrospectively collected from ten medical centers between 2011 and 2018. The original study was conducted in accordance with the Declaration of Helsinki and informed consent was waived due to the retrospective nature of the study. For the current study, the baseline staging T2WI of 705 patients (from 9 centers) was selected from this previous dataset. Patients were excluded if they had one of the following properties: non-diagnostic image quality issues, multiple tumors in the field of view, abscesses surrounding the tumor, unavailability of pre-treatment T2WI, or the unavailability of axial images. To train the anatomical inpainting model for reconstructing the healthy rectum and mesorectum, 300 samples with healthy rectum and mesorectum were randomly selected from the PICAI dataset. Table 1 shows the patient characteristics of all the selective patients including age, gender, clinical T and N staging, tumor location, and EMVI status. View this table: View inline View popup Download powerpoint Table 1 Summary of patient demographic and clinical characteristics of the multicenter dataset 4.2 Ground Truth Segmentation The ground truth segmentation labels were annotated by a GIT radiologist (M.A.A) with 6-7 years of radiological experience in interpreting rectal MRI. Segmentation masks for the rectum and mesorectum were annotated using 180 randomly selected rectal cases and 100 prostate T2W images. Specifically, the entire rectum was annotated, including its lumen from the anorectal junction to the recto-sigmoid junction, following the definition of the sigmoid take-off as the upper anatomical landmark of the rectum. The mesorectal fat, enveloped by the mesorectal fascia, was identified as the high T2 signal area surrounding the rectum on all sides, thinner anteriorly than postero-laterally ( FACG, 2006 ; Lambregts et al., 2010 ). Moving caudally towards the lower rectum, the thickness of the mesorectal fat enveloping the rectal wall decreases due to the gradual tapering of the mesorectum ( Lambregts et al., 2010 ). Tumor segmentation was annotated for all the cases (n = 705), where tumors were labeled as abnormal mural growth within the rectal lumen and extending outside into the mesorectum ( Bogveradze et al., 2023 ; Lambregts et al., 2010 ). See Table S2 for annotation details. 4.3 Rectal Anomaly Detection According to Table 2 , the inpainting model demonstrated superior performance and less variance in reconstructing the rectum and mesorectum in prostate T2WI, with an averaged SSIM (aSSIM) of 86.72 and an averaged PSNR (aPSNR) of 25.87, compared to rectal T2WI, which had an aSSIM of 83.38 and an aPSNR of 23.87. These results aligned with expectations. Prostate T2WI was in-distribution and represented healthy samples of the rectum and mesorectum. Conversely, rectal T2WI with tumor regions were out-of-distribution data and exhibited greater discrepancies from their corresponding pseudo-healthy counterparts. Importantly, we observed that aSSIM, 39.32, and aPSNR, 17.50 of tumoral regions are noticeably lower than the tumor-free regions, indicating effective anomaly detection. The anomaly map can be generated from the differences between original and pseudo-healthy images, see Figure 4 . Tumoral regions were highlighted in rectal T2WI slices. Download figure Open in new tab Figure 4: The visualization of anatomical (including rectum and mesorectum) inpainting. The columns from left to right are original T2WI slices, masked slices, inpainted slices, and anomaly map, which is the difference between original and reconstructed slices. The first two rows are prostate T2WI and the last two rows are from rectal T2WI. View this table: View inline View popup Download powerpoint Table 2 Rectum and Mesorectum inpainting performance in the test data 4.4 The comparison of Nine DL models for Rectal Tumor Segmentation on the external test dataset In the internal dataset, MedFormer delivered the best overall performance, with an average DSC (aDSC) of 66.3 and a median 95% HD (mHD) of 6.39 mm. U-mamba achieved the highest median DSC (mDSC), while ResUNet excelled in the average 95% HD, see Table S3. However, in the external testing data, as is displayed in Table 3 , S4, and Figure 5a , nnUNet achieved the best results, with aDSC 62.8 and aHD 17.28 mm, significantly better than other models. We observed that nnUNet consistently outperformed other models in the external set when the training dataset was increased to 132 from a single center, see Table S10, S11. Additionally, recently proposed transformer-based networks including UNetR, SwinUNet, and nnFormer or the SSMs-inspired architecture like U-Mamba underperformed with respect to CNN-based architectures in external data. We also compared the size of trainable parameters and Floating Point Operations (FLOPs) for each model, see Figure 5b . In summary, nnUNet demonstrated the highest DSC despite having relatively few model parameters and a low number of FLOPs. From Figure 6 , nnUNet showed better prediction performance in identifying rectal tumor regions across various centers when using multi-center T2WI. nnUNet exhibited a significantly reduced false positive voxel classification in these displaying cases, especially in the last row of Figure 6 , which was a tumor-free slice. Download figure Open in new tab Figure 5: (a) The DSC and 95% HD boxplots of various DL models were compared. Yellow triangle denotes the mean DSC. (b) DSC v.s. number of trainable parameters v.s. Flops. y-axis: DSC, x-axis: number of parameters/M, bubble size/number under model: the Flops/G. Flops: Floating Point Operations Per Second. Download figure Open in new tab Figure 6: The visualization of the segmentation performance of all nine DL models using T2WI. Each row is a different sample from the external set. The columns from left to right are original T2WI, GT: ground truth, tumor prediction masks from UNet, ResUNet, UNetR, SwinUNetR, Atten-UNet, MedFormer, nnFormer, U-Mamba, nnUNet. View this table: View inline View popup Download powerpoint Table 3 Comparison of various models on rectal tumor segmentation in the external test (Num = 666, 9 centers) 4.5 Rectum and Mesorectum Segmentation nnUNet demonstrated superior performance for rectum segmentation with an aDSC of 0.87 and mHD 10.15 mm, better than the mesorectum segmentation with an aDSC of 0.81, aHD 10.57 mm in the external cohort containing 141 samples see Table S7, Figure S1. This disparity in performance between the rectum and mesorectum is attributed to the rectum’s more regular shape than that of the mesorectum. This is consistent with the findings by ( Hamabe et al., 2022 ). Overall, the segmentation tasks for rectal anatomical structures were significantly easier than for tumors. 4.6 MTnnUNet, MCnnUNet and AAnnUNet From Table 4 and Figure 7a , MTnnUNet and AAnnUNet significantly outperformed both nnUNet and MCnnUNet. Even though MCnnUNet showed the best results in the internal validation set (see Table S8), it exhibited the lowest aDSC and aHD in the external test. The reason could be that MCnnUNet depends more on prior anatomical knowledge and there was substantial inconsistency of anatomical structures in training and inference. MCnnUNet was initially trained with ground truth anatomical structure inputs but utilized AI-annotated structures during inference, causing a performance drop in the external test set. Unlike MCnnUNet, MTnnUNet required anatomical knowledge only during training and did not rely on it during inference. Although AAnnUNet, fusing anomaly maps with highlighted tumoral regions, also relied on the quality of rectum and mesorectum masks, it yielded slightly better results than MTnnUNet in terms of aDSC. Different from MCnnUNet, which incorporated anatomical masks in a straightforward manner, AAnnUNet was provided with the highlighted anomalous areas derived from healthy distributions. We also noted many false-negative cases, where the predicted tumor mask was empty. To address this issue, we combined MTnnUNet and AAnnUNet using a union ensemble approach. This ensemble method improved performance by 3% in aDSC but did not enhance HD, as the union approach did not reduce false positives. Figure 8 illustrates that AAnnUNet performed well in tumor segmentation for both small and large tumors, highlighting the effectiveness of anomaly fusion. However, when anomaly maps were inadequate, AAnnUNet struggled to detect tumors, as shown in the last row of Figure 8 . Additionally, there were cases where all models failed to segment the tumor. However, anomaly maps can signal potential issues with high intensity values, as depicted in Figure 9 . Download figure Open in new tab Figure 7: (a) The DSC and 95% HD boxplots of nnUNet, MTnnUNet, and MCnnUNet in a fully supervised manner. (b) The DSC 95% HD boxplots of nnUNet, MTnnUNet, and MCnnUNet in a semi-supervised manner. Download figure Open in new tab Figure 8: The visualization of the segmentation performance of nnUNet, MTnnUNet, MCnnUNet, AAnnUNet, and Ensemble using T2WI, supervised setting. Each row is a different sample from the external test set. The columns from left to right are original T2WI, ground truth, tumor prediction masks from nnUNet, MTnnUNet, MCnnUNet, AAnnUNet, and Ensemble. Download figure Open in new tab Figure 9: The example where all algorithms failed to locate and segment the rectal tumor, but the anomaly map highlighted the tumoral region. View this table: View inline View popup Download powerpoint Table 4 Comparison of nnUNet, MTnnUNet, MCnnUNet, AannUNet and Ensemble on rectal tumor segmentation in the external test, fully supervised setting (Num = 666, 9 centers) 4.7 Semi-supervised MTnnUNet, MCnnUNet and AAnnUNet In the semi-supervised setting, AI-generated pseudo rectum and mesorectum segmentation masks were incorporated during training using training cohort 2, where 141 cases from one center were included for MTnnUNet, MCnnUNet, and AAnnUNet. In the internal validation set, we observed that MTnnUNet showed overall the best results. However, AAnnUNet demonstrated the best results in the external test. In contrast to the purely supervised experiments, MCnnUNet showed better aDSC and aHD in the external set. The reason could be that MCnnUNet was trained and inferred using AI-generated pseudo rectum and mesorectum priors, showing more consistency in performance between internal validation and external test, as shown in Table 5 , Table S9, Figure 7b . View this table: View inline View popup Download powerpoint Table 5 Comparison of nnUNet, MTnnUNet, MCnnUNet, AAnnUNet and Ensemble on rectal tumor segmentation in the external test, semi-supervised setting (Num = 564, 8 centers) In both supervised and semi-supervised settings, we observed that MTnnUNet exhibited better tumor segmentation results than the baseline model. MCnnUNet demonstrated improvement in the semi-supervised setting, a result that was not observed in the supervised setting. More importantly, the fusion of anomaly maps in the segmentation model ensures more precise tumor localization in both settings. In the first row of Figure S2, nnUNet mistook part of the bladder as the tumor region due to the similar intensity values, and with information fusion from either anatomical masks or anomaly maps, MTnnUNet, MCnnUNet, and AAnnUNet correctly classified the tumor regions. 5. Discussion In this study, we successfully developed a rectal anomaly detection model by training the anatomical inpainting model using prostate T2WI. The model was then applied to the rectal T2WI to generate pseudo-healthy rectal strucures and the anomaly map was the differences between the pseudo-healthy and origial rectal structures. The derived anomaly maps were then used in the downstream tumor segmentation tasks and outperformed the baseline nnUNet, MTnnUNet, which augmented training tasks with the inclusion of rectum and mesorectum segmentation masks, and MCnnunet including rectum and mesorectum as additional input channels, in both the fully-supervised and the semi-supervised manners. In order to automatically generate the rectum and mesorectum mask, we trained an nnUNet to effectively delineate anatomical structures including the normal rectum (without tumoral involvement) and mesorectum. Research in medical image analysis using AI bears many promises to improve patients’ health. However, Varoquaux et al. ( Varoquaux and Cheplygina, 2022 ) have pointed out that in academia, even though the goal is to solve scientific problems, the emphasis on publication quantity is influenced by Goodhart’s law ( Teney et al., 2020 ), which can compromise scientific quality. Researchers, in pursuit of novelty, may introduce unnecessary complexity in methods, contributing to technical debt without substantial improvement in predictions. ( Isensee et al., 2024 ) has conducted an extensive benchmarking of current segmentation methods across different dataset and their results revealed a concerning trend that most models introduced in recent years fail to outperform the nnUNet. This is consistent with the findings of this study. Recently published models with higher complexity did not exhibit higher tumor segmentation performance or better generalization ability. Although ViT and SSMs have demonstrated promising results in the natural image classification ( Xiao et al., 2023 ), in rectal tumor contouring using T2WI, transformer-based networks or SSMs-inspired networks do not show better results than CNN-based architectures. The reasons could be, that the transformer relies heavily on large-scale training and remains inferior to CNNs when training data is scarce. Unlike natural images, medical datasets are smaller, typically in the hundreds or thousands ( Willemink et al., 2020 ). Secondly, the quadratic complexity of self-attention poses challenges when dealing with long token sequences ( Hatamizadeh et al., 2022a ), especially for 3D T2WI. Beyond architectural optimization, a profound comprehension of medical imaging is integral to advancing tumor segmentation. The rectum and mesorectum provide crucial anatomical context for the localization and delineation of tumors. Several studies ( Defeudis et al., 2022 ; Jayaprakasam et al., 2022 ) have proved that mesorectal fat and tumor neighboring regions contain important prognostic information in patients with rectal cancer. Therefore, there is a substantial demand for a rectum and mesorectum segmentation model that is accurate and robust. While previous studies ( DeSilvio et al., 2023 ; Hamabe et al., 2022 ; Kim et al., 2019 ; Lee et al., 2019 ) have proposed segmentation networks for these structures, an extensive multi-center external test is highly desirable, and the majority employed 2D models. In this study, we externally tested the anatomical 3D nnUNet and observed successful rectal structure contouring. Currently, most rectal tumor segmentation studies rely on retrospective datasets, which do not provide a healthy representation of the rectum. Abdominal imaging, particularly prostate MRI, shares overlapping anatomical structures with rectal MRI, and most prostate MRIs display healthy rectal structures. As a cross-domain application, we utilized a public prostate dataset to train the inpainting model, allowing it to learn the distribution of a healthy rectum and mesorectum and generate rectal anomaly maps. Compared to simpler anatomical fusion approaches like MCnnUNet and MTnnUNet, AAnnUNet demonstrated a deeper understanding of rectal and mesorectal anatomy, resulting in improved tumor localization. The anomaly maps also have the potential for other clinical downstream tasks, like lymph node diagnosis, treatment response prediction, and staging stratification. There are some limitations of this study. First, all the T2WI was annotated by one radiologist. Multiple readers would add extra value to our research. Second, this study exclusively included T2W-MRI, with scans from 2011 to 2018. Some of them exhibited relatively low resolution and larger slice thicknesses, which would not meet the high-resolution criteria established by current protocol recommendations, see Table S1. Third, other MRI sequences like DWI and ADC can potentially improve the segmentation performance. Dou et al. ( Dou et al., 2023 ) have demonstrated by combining T2WI, ADC, and DWI, their model showed the best tumor segmentation performance. Fourth, although we have a relatively large and heterogeneous cohort, the data was solely from the Netherlands. Fifth, concerning rectal anatomical structures, the segmentation process focused solely on the rectum and mesorectum. In future studies, other anatomical structures such as the lumen could contribute to the performance. Lastly, the training and testing were performed exclusively on pre-treatment T2WI. The rectal environment in pre-treatment MRI is visually and pathologically distinct from post-treatment MRI. Incorporating post-treatment T2WI can enhance tumor characterization and improve downstream analytical workflow. 6. Conclusion In this study, we proposed an anatomical inpainting model using prostate MRI and generated rectal anomaly maps from the difference between the original rectal images and reconstructed pseudo healthy slices. We observed that rectal anomaly maps were highly associated with the tumoral areas. To explore the effect of these anomaly maps in the downstream tasks, they were fused as additional input in the rectal tumor segmentation task (AAnnUNet). Other manners of anatomical knowledge fusion including MTnnUNet and MCnnUNet in tumor segmentation were compared. As part of this research, we benchmarked nine DL models as baselines, including CNN-based, transformer-based, and Mamba-based architectures, for rectal tumor segmentation on a large multicenter dataset. nnUNet achieved the best results in the external test set, despite having a relatively lower model complexity compared to the other models, indicating that increased model complexity does not guarantee improved performance. By fusing anatomical knowledge into the tumor segmentation model, MTnnUNet, MCnnUNet, and AAnnUNet demonstrated improved performance compared to nnUNet. Among these, AAnnUNet achieved the highest accuracy in rectal tumor segmentation. Anomaly map generation requires segmentation masks of the rectum and mesorectum. We trained an nnUNet for this, achieving high accuracy with an aDSC over 0.80 on the external dataset. MTnnUNet, MCnnUNet, and AAnnUNet, when trained with AI-generated pseudo rectum and mesorectum masks, significantly improved rectal tumor segmentation performance compared to the baseline model, with AAnnUNet achieving the best results in tumor segmentation. Our analysis indicates that integrating anomaly maps derived from inpainting can effectively enhance tumor segmentation performance. Additionally, the anomaly maps, which are strongly associated with rectal tumors, can be utilized in other downstream tasks to actively monitor rectal cancer. The code, model weights and the manually annotated rectum masks for prostate T2WI and can be accessed here: https://github.com/Liiiii2101/Anatomy-aware-nnUNet-for-Rectal-Tumor-Segmentation . Data Availability All code, trained weights and annotated prostate rectum and mesorectum masks will be made publicly available upon publication upon https://github.com/Liiiii2101/Anatomy-aware-nnUNet-for-Rectal-Tumor-Segmentation . The original data of rectal cancer patient is private and is not publicly available to guarantee protection of patient privacy. https://github.com/Liiiii2101/Anatomy-aware-nnUNet-for-Rectal-Tumor-Segmentation Funding sources This study has received funding from the European Union’s Horizon 2020 Research and Innovation Programme under the Marie Skłodowska-Curie grant agreement No 857894. Acknowledgment The study was supported by the Research High Performance Computing (RHPC) facility of the Netherlands Cancer Institute. Footnotes ↵ † Shared last author Reference 1. ↵ Baur , C. , Denner , S. , Wiestler , B. , Navab , N. , Albarqouni , S ., 2021 . Autoencoders for unsupervised anomaly segmentation in brain MR images: A comparative study . Med Image Anal 69 , 101952 . doi: 10.1016/j.media.2020.101952 OpenUrl CrossRef PubMed 2. ↵ Beets-Tan , R.G. , Lambregts , D.M. , Maas , M. , Bipat , S. , Barbaro , B. , Curvo-Semedo , L. , Fenlon , H.M. , Gollub , M.J. , Gourtsoyianni , S. , Halligan , S ., 2018 . Magnetic resonance imaging for clinical management of rectal cancer: updated recommendations from the 2016 European Society of Gastrointestinal and Abdominal Radiology (ESGAR) consensus meeting . European radiology 28 , 1465 – 1475 . OpenUrl CrossRef PubMed 3. ↵ Bogveradze , N. , el Khababi , N. , Schurink , N.W. , van Griethuysen , J.J.M. , de Bie , S. , Bosma , G. , Cappendijk , V.C. , Geenen , R.W.F. , Neijenhuis , P. , Peterson , G. , Veeken , C.J. , Vliegen , R.F.A. , Maas , M. , Lahaye , M.J. , Beets , G.L. , Beets-Tan , R.G.H. , Lambregts , D.M.J. , 2022 . Evolutions in rectal cancer MRI staging and risk stratification in The Netherlands . Abdom Radiol 47 , 38 – 47 . doi: 10.1007/s00261-021-03281-8 OpenUrl CrossRef 4. ↵ Bogveradze , N. , Snaebjornsson , P. , Grotenhuis , B.A. , van Triest , B. , Lahaye , M.J. , Maas , M. , Beets , G.L. , Beets-Tan , R.G.H. , Lambregts , D.M.J. , 2023 . MRI anatomy of the rectum: key concepts important for rectal cancer staging and treatment planning . Insights into Imaging 14 , 13 . doi: 10.1186/s13244-022-01348-8 OpenUrl CrossRef PubMed 5. ↵ Cai , L. , Lambregts , D.M.J. , Beets , G.L. , Mass , M. , Pooch , E.H.P. , Guérendel , C. , Beets-Tan , R.G.H. , Benson , S ., 2024 . An automated deep learning pipeline for EMVI classification and response prediction of rectal cancer using baseline MRI: a multi-centre study. npj Precis . Onc . 8 , 1 – 11 . doi: 10.1038/s41698-024-00516-x OpenUrl CrossRef 6. ↵ Chen , X. , You , S. , Tezcan , K.C. , Konukoglu , E. , 2020 . Unsupervised Lesion Detection via Image Restoration with a Normative Prior . 7. ↵ Ourselin , S. , Joskowicz , L. , Sabuncu , M.R. , Unal , G. , Wells , W Çiçek , Ö. , Abdulkadir , A. , Lienkamp , S.S. , Brox , T. , Ronneberger , O ., 2016 . 3D U-Net: Learning Dense Volumetric Segmentation from Sparse Annotation , in: Ourselin , S. , Joskowicz , L. , Sabuncu , M.R. , Unal , G. , Wells , W . (Eds.), Medical Image Computing and Computer-Assisted Intervention – MICCAI 2016 . Springer International Publishing , Cham , pp. 424 – 432 . doi: 10.1007/978-3-319-46723-8_49 OpenUrl CrossRef 8. ↵ Defeudis , A. , Mazzetti , S. , Panic , J. , Micilotta , M. , Vassallo , L. , Giannetto , G. , Gatti , M. , Faletti , R. , Cirillo , S. , Regge , D ., 2022 . MRI-based radiomics to predict response in locally advanced rectal cancer: Comparison of manual and automatic segmentation on external validation in a multicentre study . European Radiology Experimental 6 , 19 . OpenUrl PubMed 9. ↵ DeSilvio , T. , Antunes , J.T. , Bera , K. , Chirra , P. , Le , H. , Liska , D. , Stein , S.L. , Marderstein , E. , Hall , W. , Paspulati , R. , Gollamudi , J. , Purysko , A.S. , Viswanath , S.E ., 2023 . Region-specific deep learning models for accurate segmentation of rectal structures on post-chemoradiation T2w MRI: a multi-institutional, multi-reader study . Front. Med . 10 . doi: 10.3389/fmed.2023.1149056 OpenUrl CrossRef 10. ↵ Diakogiannis , F.I. , Waldner , F. , Caccetta , P. , Wu , C ., 2020 . ResUNet-a: A deep learning framework for semantic segmentation of remotely sensed data . ISPRS Journal of Photogrammetry and Remote Sensing 162 , 94 – 114 . doi: 10.1016/j.isprsjprs.2020.01.013 OpenUrl CrossRef 11. ↵ Dou , M. , Chen , Z. , Tang , Y. , Sheng , L. , Zhou , J. , Wang , X. , Yao , Y ., 2023 . Segmentation of rectal tumor from multi-parametric MRI images using an attention-based fusion network . Med Biol Eng Comput 61 , 2379 – 2389 . doi: 10.1007/s11517-023-02828-9 OpenUrl CrossRef PubMed 12. ↵ El Khababi , N. , Beets-Tan , R.G.H. , Tissier , R. , Lahaye , M.J. , Maas , M. , Curvo-Semedo , L. , Dresen , R.C. , Nougaret , S. , Beets , G.L. , Lambregts , D.M.J. , Bakers , F.C.H. , Barros , P. , Bauer , F. , de Bie , S.H. , Ballantyne , S. , Dutra , J.B. , Buskov , L. , Bogveradze , N. , Bosma , G.P.T. , Cappendijk , V.C. , Castagnoli , F. , Charalampos , S. , Delli Pizzi , A. , Digby , M. , Geenen , R.W.F. , van Griethuysen , J.J.M. , Lafrance , J. , Mahajan , V. , Malekzadeh , S. , Neijenhuis , P.A. , Peterson , G.M. , Pieters , I. , Schurink , N.W. , Smit , R. , Veeken , C.J. , Vliegen , R.F.A. , Wray , A. , Zeina , A.-R. , on behalf of the rectal MRI study group , 2023 . Predicting response to chemoradiotherapy in rectal cancer via visual morphologic assessment and staging on baseline MRI: a multicenter and multireader study . Abdom Radiol 48 , 3039 – 3049 . doi: 10.1007/s00261-023-03961-7 OpenUrl CrossRef 13. ↵ ESGAR 2020 Book of Abstracts , 2020 . . Insights into Imaging 11 , 64 . doi: 10.1186/s13244-020-00873-8 OpenUrl CrossRef PubMed 14. ↵ FACG, M.A.M., MD, FACR , 2006 . Dynamic Radiology of the Abdomen: Normal and Pathologic Anatomy . Springer Science & Business Media . 15. ↵ Gao , Y. , Zhou , M. , Liu , D. , Yan , Z. , Zhang , S. , Metaxas , D.N ., 2023 . A Data-scalable Transformer for Medical Image Segmentation: Architecture , Model Efficiency, and Benchmark . doi: 10.48550/arXiv.2203.00131 OpenUrl CrossRef 16. ↵ Gong , D. , Liu , L. , Le , V. , Saha , B. , Mansour , M.R. , Venkatesh , S. , Van Den Hengel , A ., 2019 . Memorizing Normality to Detect Anomaly: Memory-Augmented Deep Autoencoder for Unsupervised Anomaly Detection , in: 2019 IEEE/CVF International Conference on Computer Vision (ICCV). Presented at the 2019 IEEE/CVF International Conference on Computer Vision (ICCV) , pp. 1705 – 1714 . doi: 10.1109/ICCV.2019.00179 OpenUrl CrossRef 17. ↵ Goodfellow , I. , Pouget-Abadie , J. , Mirza , M. , Xu , B. , Warde-Farley , D. , Ozair , S. , Courville , A. , Bengio , Y ., 2014 . Generative Adversarial Nets, in: Advances in Neural Information Processing Systems . Curran Associates, Inc . 18. ↵ Gu , A. , Johnson , I. , Goel , K. , Saab , K. , Dao , T. , Rudra , A. , Ré , C ., 2021 . Combining Recurrent, Convolutional, and Continuous-time Models with Linear State Space Layers, in: Advances in Neural Information Processing Systems . Curran Associates, Inc ., pp. 572 – 585 . 19. ↵ Hamabe , A. , Ishii , M. , Kamoda , R. , Sasuga , S. , Okuya , K. , Okita , K. , Akizuki , E. , Sato , Y. , Miura , R. , Onodera , K. , Hatakenaka , M. , Takemasa , I ., 2022 . Artificial intelligence–based technology for semi-automated segmentation of rectal cancer using high-resolution MRI . PLoS One 17 , e0269931 . doi: 10.1371/journal.pone.0269931 OpenUrl CrossRef PubMed 20. ↵ Han , L. , Tan , T. , Zhang , T. , Huang , Y. , Wang , X. , Gao , Y. , Teuwen , J. , Mann , R ., 2024 . Synthesis-based imaging-differentiation representation learning for multi-sequence 3D/4D MRI . Med Image Anal 92 , 103044 . doi: 10.1016/j.media.2023.103044 OpenUrl CrossRef PubMed 21. ↵ Han , L. , Zhang , T. , Huang , Y. , Dou , H. , Wang , X. , Gao , Y. , Lu , C. , Tan , T. , Mann , R. , 2023 . An Explainable Deep Framework: Towards Task-Specific Fusion for Multi-to-One MRI Synthesis , in: Greenspan , H. , Madabhushi , A. , Mousavi , P. , Salcudean , S. , Duncan , J. , Syeda-Mahmood , T. , Taylor , R. (Eds.), Medical Image Computing and Computer Assisted Intervention – MICCAI 2023 . Springer Nature Switzerland , Cham , pp. 45 – 55 . doi: 10.1007/978-3-031-43999-5_5 OpenUrl CrossRef 22. ↵ Crimi , A. , Bakas , S Hatamizadeh , A. , Nath , V. , Tang , Y. , Yang , D. , Roth , H.R. , Xu , D ., 2022a . Swin UNETR: Swin Transformers for Semantic Segmentation of Brain Tumors in MRI Images , in: Crimi , A. , Bakas , S . (Eds.), Brainlesion: Glioma, Multiple Sclerosis, Stroke and Traumatic Brain Injuries . Springer International Publishing , Cham , pp. 272 – 284 . doi: 10.1007/978-3-031-08999-2_22 OpenUrl CrossRef 23. ↵ Hatamizadeh , A. , Tang , Y. , Nath , V. , Yang , D. , Myronenko , A. , Landman , B. , Roth , H.R. , Xu , D ., 2022b . UNETR: Transformers for 3D Medical Image Segmentation . Presented at the Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision , pp. 574 – 584 . 24. ↵ Leibe , B. , Matas , J. , Sebe , N. , Welling , M He , K. , Zhang , X. , Ren , S. , Sun , J ., 2016 . Identity Mappings in Deep Residual Networks , in: Leibe , B. , Matas , J. , Sebe , N. , Welling , M . (Eds.), Computer Vision – ECCV 2016 . Springer International Publishing , Cham , pp. 630 – 645 . doi: 10.1007/978-3-319-46493-0_38 OpenUrl CrossRef 25. ↵ Hearn , N. , Bugg , W. , Chan , A. , Vignarajah , D. , Cahill , K. , Atwell , D. , Lagopoulos , J. , Min , M ., 2020 . Manual and semi-automated delineation of locally advanced rectal cancer subvolumes with diffusion-weighted MRI . Br J Radiol 93 , 20200543 . doi: 10.1259/bjr.20200543 OpenUrl CrossRef PubMed 26. ↵ Horvat , N. , Carlos Tavares Rocha , C. , Clemente Oliveira , B. , Petkovska , I. , Gollub , M.J. , 2019 . MRI of rectal cancer: tumor staging, imaging techniques, and management . Radiographics 39 , 367 – 387 . OpenUrl PubMed 27. ↵ Irving , B. , Franklin , J.M. , Papież , B.W. , Anderson , E.M. , Sharma , R.A. , Gleeson , F.V. , Brady , S.M. , Schnabel , J.A ., 2016 . Pieces-of-parts for supervoxel segmentation with global context: Application to DCE-MRI tumour delineation . Medical Image Analysis 32 , 69 – 83 . doi: 10.1016/j.media.2016.03.002 OpenUrl CrossRef PubMed 28. ↵ Isensee , F. , Jaeger , P.F. , Kohl , S.A. , Petersen , J. , Maier-Hein , K.H ., 2021 . nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation . Nature methods 18 , 203 – 211 . OpenUrl PubMed 29. ↵ Isensee , F. , Wald , T. , Ulrich , C. , Baumgartner , M. , Roy , S. , Maier-Hein , K. , Jaeger , P.F. , 2024 . nnU-Net Revisited: A Call for Rigorous Validation in 3D Medical Image Segmentation . 30. ↵ Jayaprakasam , V.S. , Paroder , V. , Gibbs , P. , Bajwa , R. , Gangai , N. , Sosa , R.E. , Petkovska , I. , Golia Pernicka , J.S. , Fuqua , J.L. , Bates , D.D ., 2022 . MRI radiomics features of mesorectal fat can predict response to neoadjuvant chemoradiation therapy and tumor recurrence in patients with locally advanced rectal cancer . European radiology 1 – 10 . 31. ↵ Jian , J. , Xiong , F. , Xia , W. , Zhang , R. , Gu , J. , Wu , X. , Meng , X. , Gao , X ., 2018 . Fully convolutional networks (FCNs)-based segmentation method for colorectal tumors on T2-weighted magnetic resonance images . Australas Phys Eng Sci Med 41 , 393 – 401 . doi: 10.1007/s13246-018-0636-9 OpenUrl CrossRef PubMed 32. ↵ Leibe , B. , Matas , J. , Sebe , N. , Welling , M Johnson , J. , Alahi , A. , Fei-Fei , L ., 2016 . Perceptual Losses for Real-Time Style Transfer and Super-Resolution , in: Leibe , B. , Matas , J. , Sebe , N. , Welling , M . (Eds.), Computer Vision – ECCV 2016 . Springer International Publishing , Cham , pp. 694 – 711 . doi: 10.1007/978-3-319-46475-6_43 OpenUrl CrossRef 33. ↵ Kim , J. , Oh , J.E. , Lee , J. , Kim , M.J. , Hur , B.Y. , Sohn , D.K. , Lee , B ., 2019 . Rectal cancer: Toward fully automatic discrimination of T2 and T3 rectal cancers using deep convolutional neural network . International Journal of Imaging Systems and Technology 29 , 247 – 259 . doi: 10.1002/ima.22311 OpenUrl CrossRef 34. ↵ Kingma , D.P. , Welling , M. , 2022 . Auto-Encoding Variational Bayes . 35. ↵ Knuth , F. , Adde , I.A. , Huynh , B.N. , Groendahl , A.R. , Winter , R.M. , Negård , A. , Holmedal , S.H. , Meltzer , S. , Ree , A.H. , Flatmark , K. , Dueland , S. , Hole , K.H. , Seierstad , T. , Redalen , K.R. , Futsaether , C.M ., 2022 . MRI-based automatic segmentation of rectal cancer using 2D U-Net on two independent cohorts . Acta Oncologica 61 , 255 – 263 . doi: 10.1080/0284186X.2021.2013530 OpenUrl CrossRef 36. ↵ Stoker , J Lambregts , D.M.J. , Maas , M. , Beets-Tan , R.G.H ., 2010 . MRI of the Rectum , in: Stoker , J . (Ed.), MRI of the Gastrointestinal Tract . Springer , Berlin, Heidelberg , pp. 205 – 227 . doi: 10.1007/978-3-540-85532-3_13 OpenUrl CrossRef 37. ↵ Lee , J. , Oh , J.E. , Kim , M.J. , Hur , B.Y. , Sohn , D.K ., 2019 . Reducing the Model Variance of a Rectal Cancer Segmentation Network . IEEE Access 7 , 182725 – 182733 . doi: 10.1109/ACCESS.2019.2960371 OpenUrl CrossRef 38. ↵ Li , D. , Wang , J. , Yang , J. , Zhao , J. , Yang , X. , Cui , Y. , Zhang , K ., 2023 . RTAU-Net: A novel 3D rectal tumor segmentation model based on dual path fusion and attentional guidance . Computer Methods and Programs in Biomedicine 242 , 107842 . doi: 10.1016/j.cmpb.2023.107842 OpenUrl CrossRef PubMed 39. ↵ Ma , J. , Li , F. , Wang , B ., 2024 . U-Mamba: Enhancing Long-range Dependency for Biomedical Image Segmentation . doi: 10.48550/arXiv.2401.04722 OpenUrl CrossRef 40. ↵ Menze , B.H. , Jakab , A. , Bauer , S. , Kalpathy-Cramer , J. , Farahani , K. , Kirby , J. , Burren , Y. , Porz , N. , Slotboom , J. , Wiest , R. , Lanczi , L. , Gerstner , E. , Weber , M.-A. , Arbel , T. , Avants , B.B. , Ayache , N. , Buendia , P. , Collins , D.L. , Cordier , N. , Corso , J.J. , Criminisi , A. , Das , T. , Delingette , H. , Demiralp , Ç. , Durst , C.R. , Dojat , M. , Doyle , S. , Festa , J. , Forbes , F. , Geremia , E. , Glocker , B. , Golland , P. , Guo , X. , Hamamci , A. , Iftekharuddin , K.M. , Jena , R. , John , N.M. , Konukoglu , E. , Lashkari , D. , Mariz , J.A. , Meier , R. , Pereira , S. , Precup , D. , Price , S.J. , Raviv , T.R. , Reza , S.M.S. , Ryan , M. , Sarikaya , D. , Schwartz , L. , Shin , H.-C. , Shotton , J. , Silva , C.A. , Sousa , N. , Subbanna , N.K. , Szekely , G. , Taylor , T.J. , Thomas , O.M. , Tustison , N.J. , Unal , G. , Vasseur , F. , Wintermark , M. , Ye , D.H. , Zhao , L. , Zhao , B. , Zikic , D. , Prastawa , M. , Reyes , M. , Van Leemput , K ., 2015 . The Multimodal Brain Tumor Image Segmentation Benchmark (BRATS) . IEEE Transactions on Medical Imaging 34 , 1993 – 2024 . doi: 10.1109/TMI.2014.2377694 OpenUrl CrossRef PubMed 41. ↵ MERCURY Study Group , 2007 . Extramural depth of tumor invasion at thin-section MR in patients with rectal cancer: results of the MERCURY study . Radiology 243 , 132 – 139 . doi: 10.1148/radiol.2431051825 OpenUrl CrossRef PubMed Web of Science 42. ↵ Mlynarski , P. , Delingette , H. , Criminisi , A. , Ayache , N ., 2019 . 3D convolutional neural networks for tumor segmentation using long-range 2D context . Computerized Medical Imaging and Graphics 73 , 60 – 72 . doi: 10.1016/j.compmedimag.2019.02.001 OpenUrl CrossRef PubMed 43. ↵ Nagtegaal , I. , Gaspar , C. , Marijnen , C. , van de Velde , C. , Fodde , R. , van Krieken , H. , 2004 . Morphological changes in tumour type after radiotherapy are accompanied by changes in gene expression profile but not in clinical behaviour . The Journal of Pathology 204 , 183 – 192 . doi: 10.1002/path.1621 OpenUrl CrossRef PubMed Web of Science 44. ↵ Nguyen , B. , Feldman , A. , Bethapudi , S. , Jennings , A. , Willcocks , C.G ., 2021 . Unsupervised Region-Based Anomaly Detection In Brain MRI With Adversarial Image Inpainting , in: 2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI) . Presented at the 2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI) , pp. 1127 – 1131 . doi: 10.1109/ISBI48211.2021.9434115 OpenUrl CrossRef 45. ↵ Oktay , O. , Schlemper , J. , Folgoc , L.L. , Lee , M. , Heinrich , M. , Misawa , K. , Mori , K. , McDonagh , S. , Hammerla , N.Y. , Kainz , B. , Glocker , B. , Rueckert , D ., 2018 . Attention U-Net: Learning Where to Look for the Pancreas . doi: 10.48550/arXiv.1804.03999 OpenUrl CrossRef 46. ↵ Pinaya , W.H.L. , Tudosiu , P.-D. , Gray , R. , Rees , G. , Nachev , P. , Ourselin , S. , Cardoso , M.J ., 2022 . Unsupervised brain imaging 3D anomaly detection and segmentation with transformers . Med Image Anal 79 , 102475 . doi: 10.1016/j.media.2022.102475 OpenUrl CrossRef PubMed 47. ↵ Dey , N. , Ashour , A.S. , Borra , S Razzak , M.I. , Naz , S. , Zaib , A ., 2018 . Deep Learning for Medical Image Processing: Overview, Challenges and the Future , in: Dey , N. , Ashour , A.S. , Borra , S . (Eds.), Classification in BioApps: Automation of Decision Making . Springer International Publishing , Cham , pp. 323 – 350 . doi: 10.1007/978-3-319-65981-7_12 OpenUrl CrossRef 48. ↵ Navab , N. , Hornegger , J. , Wells , W.M. , Frangi , A.F Ronneberger , O. , Fischer , P. , Brox , T ., 2015 . U-Net: Convolutional Networks for Biomedical Image Segmentation , in: Navab , N. , Hornegger , J. , Wells , W.M. , Frangi , A.F . (Eds.), Medical Image Computing and Computer-Assisted Intervention – MICCAI 2015 , Lecture Notes in Computer Science. Springer International Publishing , Cham , pp. 234–241. doi: 10.1007/978-3-319-24574-4_28 OpenUrl CrossRef PubMed 49. ↵ Saha , A. , Bosma , J. , Twilt , J. , Ginneken , B. van , Yakar , D. , Elschot , M. , Veltman , J. , Fütterer , J. , Rooij , M. de , Huisman , H. , 2023 . Artificial Intelligence and Radiologists at Prostate Cancer Detection in MRI — The PI-CAI Challenge . Presented at the Medical Imaging with Deep Learning, short paper track . 50. ↵ Schurink , N.W. , van Kranen , S.R. , Roberti , S. , van Griethuysen , J.J.M. , Bogveradze , N. , Castagnoli , F. , el Khababi , N. , Bakers , F.C.H. , de Bie , S.H. , Bosma , G.P.T. , Cappendijk , V.C. , Geenen , R.W.F. , Neijenhuis , P.A. , Peterson , G.M. , Veeken , C.J. , Vliegen , R.F.A. , Beets-Tan , R.G.H. , Lambregts , D.M.J. , 2022 . Sources of variation in multicenter rectal MRI data and their effect on radiomics feature reproducibility . Eur Radiol 32 , 1506 – 1516 . doi: 10.1007/s00330-021-08251-8 OpenUrl CrossRef PubMed 51. ↵ Schurink , N.W. , van Kranen , S.R. , van Griethuysen , J.J. , Roberti , S. , Snaebjornsson , P. , Bakers , F.C. , de Bie , S.H. , Bosma , G.P. , Cappendijk , V.C. , Geenen , R.W. , 2023 . Development and multicenter validation of a multiparametric imaging model to predict treatment response in rectal cancer . European Radiology 1 – 10 . 52. ↵ Suzuki , C. , Torkzad , M.R. , Tanaka , S. , Palmer , G. , Lindholm , J. , Holm , T. , Blomqvist , L ., 2008 . The importance of rectal cancer MRI protocols on iInterpretation accuracy . World J Surg Onc 6 , 89 . doi: 10.1186/1477-7819-6-89 OpenUrl CrossRef 53. ↵ Teney , D. , Abbasnejad , E. , Kafle , K. , Shrestha , R. , Kanan , C. , van den Hengel , A. , 2020 . On the Value of Out-of-Distribution Testing: An Example of Goodhart’ s Law, in: Advances in Neural Information Processing Systems . Curran Associates, Inc ., pp. 407 – 417 . 54. ↵ Trebeschi , S. , van Griethuysen , J.J.M. , Lambregts , D.M.J. , Lahaye , M.J. , Parmar , C. , Bakers , F.C.H. , Peters , N.H.G.M. , Beets-Tan , R.G.H. , Aerts , H.J.W.L. , 2017 . Deep Learning for Fully-Automated Localization and Segmentation of Rectal Cancer on Multiparametric MR . Sci Rep 7 , 5301 . doi: 10.1038/s41598-017-05728-9 OpenUrl CrossRef PubMed 55. ↵ van den Oord , A. , Vinyals , O. , kavukcuoglu , koray , 2017 . Neural Discrete Representation Learning, in: Advances in Neural Information Processing Systems . Curran Associates, Inc . 56. ↵ Varoquaux , G. , Cheplygina , V ., 2022 . Machine learning for medical imaging: methodological failures and recommendations for the future. npj Digit . Med . 5 , 1 – 8 . doi: 10.1038/s41746-022-00592-y OpenUrl CrossRef 57. ↵ Wang , J. , Lu , J. , Qin , G. , Shen , L. , Sun , Y. , Ying , H. , Zhang , Z. , Hu , W ., 2018 . Technical Note: A deep learning-based autosegmentation of rectal tumors in MR images . Medical Physics 45 , 2560 – 2564 . doi: 10.1002/mp.12918 OpenUrl CrossRef PubMed 58. ↵ Willemink , M.J. , Koszek , W.A. , Hardell , C. , Wu , J. , Fleischmann , D. , Harvey , H. , Folio , L.R. , Summers , R.M. , Rubin , D.L. , Lungren , M.P ., 2020 . Preparing Medical Imaging Data for Machine Learning . Radiology 295 , 4 – 15 . doi: 10.1148/radiol.2020192224 OpenUrl CrossRef PubMed 59. ↵ Woo , B. , Engstrom , C. , Baresic , W. , Fripp , J. , Crozier , S. , Chandra , S.S ., 2024 . Automated anomaly-aware 3D segmentation of bones and cartilages in knee MR images from the Osteoarthritis Initiative . Med Image Anal 93 , 103089 . doi: 10.1016/j.media.2024.103089 OpenUrl CrossRef PubMed 60. ↵ Xiao , H. , Li , L. , Liu , Q. , Zhu , X. , Zhang , Q ., 2023 . Transformers in medical image segmentation: A review . Biomedical Signal Processing and Control 84 , 104791 . doi: 10.1016/j.bspc.2023.104791 OpenUrl CrossRef 61. ↵ Yeganeh , Y. , Farshad , A. , Navab , N. , 2022 . Shape-Aware Masking for Inpainting in Medical Imaging . 62. ↵ Zhou , H.-Y. , Guo , J. , Zhang , Y. , Han , X. , Yu , L. , Wang , L. , Yu , Y ., 2023 . nnFormer: Volumetric Medical Image Segmentation via a 3D Transformer . IEEE Transactions on Image Processing 32 , 4036 – 4045 . doi: 10.1109/TIP.2023.3293771 OpenUrl CrossRef PubMed 63. ↵ Zhu , J.-Y. , Park , T. , Isola , P. , Efros , A.A ., 2017 . Unpaired Image-to-Image Translation Using Cycle-Consistent Adversarial Networks , in: 2017 IEEE International Conference on Computer Vision (ICCV) . Presented at the 2017 IEEE International Conference on Computer Vision (ICCV) , pp. 2242 – 2251 . doi: 10.1109/ICCV.2017.244 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted October 16, 2024. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study Lishan Cai , Mohamed A. Abdelatty , Luyi Han , Doenja M. J. Lambregts , Joost van Griethuysen , Eduardo Pooch , Regina G.H. Beets-Tan , Sean Benson , Joren Brunekreef , Jonas Teuwen medRxiv 2024.10.15.24315517; doi: https://doi.org/10.1101/2024.10.15.24315517 Share This Article: Copy Citation Tools Improving Rectal Tumor Segmentation with Anomaly Fusion Derived from Anatomical Inpainting: A Multicenter Study Lishan Cai , Mohamed A. Abdelatty , Luyi Han , Doenja M. J. Lambregts , Joost van Griethuysen , Eduardo Pooch , Regina G.H. Beets-Tan , Sean Benson , Joren Brunekreef , Jonas Teuwen medRxiv 2024.10.15.24315517; doi: https://doi.org/10.1101/2024.10.15.24315517 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (573) Allergy and Immunology (865) Anesthesia (304) Cardiovascular Medicine (4457) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (610) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15244) Forensic Medicine (30) Gastroenterology (1132) Genetic and Genomic Medicine (6621) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4557) Health Policy (1372) Health Systems and Quality Improvement (1615) Hematology (543) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15936) Intensive Care and Critical Care Medicine (1106) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6635) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3348) Ophthalmology (980) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5463) Public and Global Health (9257) Radiology and Imaging (2210) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1198) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (532) Surgery (714) Toxicology (100) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a0373e2c9e18593a',t:'MTc4MDA3NTk5Mw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();