Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology

Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology Ranny Rahaningrum Herdiantoputri , Daisuke Komura , Mieko Ochi , Yuki Fukawa , Kou Kayamori , Maiko Tsuchiya , Yoshinao Kikuchi , Tetsuo Ushiku , Tohru Ikeda , Shumpei Ishikawa doi: https://doi.org/10.1101/2024.05.30.24308181 Ranny Rahaningrum Herdiantoputri 1 Department of Oral Pathology, Graduate School of Medical and Dental Sciences, Tokyo Medical and Dental University , 1-5-45, Yushima, Bunkyo-ku, 1138510, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Daisuke Komura 2 Department of Preventive Medicine, Graduate School of Medicine, The University of Tokyo , 7-3-1 Hongo, Bunkyo-ku, 1130033, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: kdais-prm{at}m.u-tokyo.ac.jp ishum-prm{at}m.u-tokyo.ac.jp Mieko Ochi 2 Department of Preventive Medicine, Graduate School of Medicine, The University of Tokyo , 7-3-1 Hongo, Bunkyo-ku, 1130033, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yuki Fukawa 1 Department of Oral Pathology, Graduate School of Medical and Dental Sciences, Tokyo Medical and Dental University , 1-5-45, Yushima, Bunkyo-ku, 1138510, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Kou Kayamori 1 Department of Oral Pathology, Graduate School of Medical and Dental Sciences, Tokyo Medical and Dental University , 1-5-45, Yushima, Bunkyo-ku, 1138510, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Maiko Tsuchiya 3 Department of Pathology, Teikyo University School of Medicine , 2-11-1 Kaga, Itabashi-ku, 1738605, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Yoshinao Kikuchi 3 Department of Pathology, Teikyo University School of Medicine , 2-11-1 Kaga, Itabashi-ku, 1738605, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tetsuo Ushiku 4 Department of Pathology, Graduate School of Medicine, The University of Tokyo , 7-3-1 Hongo, Bunkyo-ku, Tokyo 1130033, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Tohru Ikeda 1 Department of Oral Pathology, Graduate School of Medical and Dental Sciences, Tokyo Medical and Dental University , 1-5-45, Yushima, Bunkyo-ku, 1138510, Tokyo, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shumpei Ishikawa 2 Department of Preventive Medicine, Graduate School of Medicine, The University of Tokyo , 7-3-1 Hongo, Bunkyo-ku, 1130033, Tokyo, Japan 5 Division of Pathology, National Cancer Center Exploratory Oncology Research & Clinical Trial Center , 6-5-1 Kashiwanoha, Kashiwa, Chiba 277-8577, Japan Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: kdais-prm{at}m.u-tokyo.ac.jp ishum-prm{at}m.u-tokyo.ac.jp Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Oral tumors necessitate a dependable computer-assisted pathological diagnosis system considering their rarity and diversity. A content-based image retrieval (CBIR) system using deep neural networks has been successfully devised for digital pathology. No CBIR system for oral pathology has been investigated because of the lack of an extensive image database and feature extractors tailored to oral pathology. This study uses a large CBIR database constructed from 30 categories of oral tumors to compare deep learning methods as feature extractors. The highest average area under the receiver operating curve (AUC) was achieved by models trained on database images using self-supervised learning (SSL) methods (0.900 with SimCLR; 0.897 with TiCo). The generalizability of the models was validated using query images from the same cases taken with smartphones. When smartphone images were tested as queries, both models yielded the highest mean AUC (0.871 with SimCLR and 0.857 with TiCo). We ensured the retrieved image result would be easily observed by evaluating the top-10 mean accuracy and checking for an exact diagnostic category and its differential diagnostic categories. Therefore, training deep learning models with SSL methods using image data specific to the target site is beneficial for CBIR tasks in oral tumor histology to obtain histologically meaningful results and high performance. This result provides insight into the effective development of a CBIR system to help improve the accuracy and speed of histopathology diagnosis and advance oral tumor research in the future. Introduction Oral tumors are generally composed of diverse and rare tumor types, except for major categories like squamous cell carcinoma. Distinguishing oral tumor types is difficult except for well-experienced oral pathologists. The rarity of oral tumors and the diverse tissue types in the oral region make obtaining reference images for diagnosis and research a challenge, potentially leading to delayed diagnosis and a significant burden on pathologists [ 1 ]. Consequently, a diagnostic system is needed to improve the speed and accuracy of histopathological diagnosis of these tumors [ 2 ]. Artificial intelligence (AI) is a promising solution for efficient histopathological diagnosis of oral tumors. AI development for oral tumor diagnosis is limited and focused only on a few tumor types. Classification methods have been developed to predict the diagnosis, such as ameloblastoma or odontogenic keratocysts, to which a histopathological image may belong [ 3 , 4 ]. These approaches are helpful in common cases. However, a computer-aided diagnostic system that covers a broader spectrum of tumor types would be more practical and would help narrow the differential diagnoses. Therefore, content-based image retrieval (CBIR) is suitable. CBIR regards histopathological images as query images to find similar images from a database based on their similar morphology [ 2 , 5 ]. This system is useful as a diagnostic aid for finding case references, especially where diagnostic expertise is challenging to find, such as in low-to middle-income countries [ 1 ]. The involvement of human intervention is crucial in diagnosis. Conventionally, pathologists diagnose directly after H&E-stained slide analysis or optionally use different methods as diagnostic aids: referring atlases, consulting subspecialist experts, or conducting ancillary tests. An automatic image search can complement these options to expedite image reference search ( Figure 1 ). With scarce pathological expertise, a tool that could provide urgently needed information for rapid diagnosis before conducting tests to raise a definitive one would be significant [ 6 ]. CBIR provides interpretability because it presents multiple candidate images, which is beneficial when distinguishing between categories based on histopathological images alone is challenging, such as when information on dental infections or radiographic findings is needed. With CBIR, the retrieved results are to be evaluated by pathologists, reducing the risk of misdiagnosis owing to inaccurate results, especially for categories with very similar histology. Download figure Open in new tab Figure 1: CBIR’s place in the pathological diagnosis workflow CBIR optionally providing an interpretable automatic reference search that is fast and easily followed up with more thorough study with atlases, discussions, or ancillary tests. CBIR could help point out similar features from the previously diagnosed image in the database that may lead to testable differential diagnoses more swiftly than directly consulting atlases or senior experts, which may cause the patient delayed treatment. (Image created with BioRender) The CBIR system consists of two aspects: image feature extraction and nearest-neighbor search. Feature extraction is crucial because it must adequately capture complex histological features such as staining patterns, tissue structures, and cellular morphology to create histologically relevant image representation [ 2 , 7 , 8 ]. The extracted features must be robust to irrelevant color variations, such as different hematoxylin and eosin (HE) stain brands, glass slide color degradation, and image-capturing devices ranging from whole-slide image (WSI) scanners to smartphone cameras [ 5 , 8 , 9 ]. At the early stage of CBIR development, traditional image features such as shape, color, texture, or a combination were used. Recent developments showed that deep learning models outperformed traditional features [ 6 , 7 ]. Several deep learning methods, such as supervised learning where models are pre-trained on general images or fine-tuned on histopathological images, have been used to train feature extractors [ 9 – 13 ], and self-supervised learning (SSL), which allows learning from unlabeled images [ 14 – 16 ]. However, no studies have reviewed which method is most suitable for CBIR in oral tumors. This study aimed to investigate the performance of different deep learning models for oral tumor CBIRs by developing a large dataset of whole-slide images from 541 cases with 51 tumor types and evaluating the retrieval accuracy by comparing different representational learning techniques. Materials & Methods Dataset We collected diagnostic slides of the oral tumor categories described in Chapters 7 and 8 of the WHO Classification of Head and Neck Tumors, 4 th Edition [ 17 ]. Patients were diagnosed in 2001–2022 and underwent surgery at Tokyo Medical and Dental University (TMDU) Hospital. Patients or their surrogates had the option to withdraw from this study through public notices according to the approved protocols. This study was approved by the Institutional Review Board (IRB) of TMDU (No. D2019-087). Some slides that were lost, broken, or required diagnostic confirmation with immunohistochemistry (IHC) staining were remade from the paraffin-embedded tissue blocks. Additional IHC staining was done for the secretory carcinoma and the atypical acinic cell carcinoma cases older than 2017. Categories with fewer than five cases were excluded. All slides that fulfill the inclusion criteria were scanned using a NanoZoomer S210 slide scanner (C13239-01; Hamamatsu Photonics, Japan) at 40× magnification. The tumor areas were annotated by a pathology resident and verified by board-certified pathologists. We included the tumor areas that are typical to the tumor category while excluding the normal tissue and the severe artifacts such as torn or folded tissue. Image patches were then randomly extracted from the annotated tumor areas with three different sizes: 905 µm, 453 µm, and 226 µm. Twenty image patches were extracted with each magnification. The use of three different sizes was to accommodate different magnification levels and preserve the histologic information at the tissue and cellular level as much as possible. The dataset comprises 49,243 image patches from 51 categories, covering approximately 50% of the oral tumor categories ( Table 1 ). View this table: View inline View popup Table 1. The tumor categories included in the dataset, their corresponding ICD-O codes, and the total number of cases Some categories do not correspond to the ICD-O but are described in the WHO Classification of Head and Neck Tumors. 4th Ed. Categories consisting of ten to twenty cases, marked in bold , were included in the CBIR database. Database construction A database from a subset of the dataset containing at least ten cases was compiled. Image representations from each model’s encoder were stored in the database ( Figure 2A ). It contains 33,356 image patches from 30 oral tumor categories ( Table 1 ). Download figure Open in new tab Figure 2: CBIR workflow and SSL models training process (A) Schematic representation of CBIR using deep neural networks to retrieve similar oral tumor histopathological images. The similarity is determined by a nearest-neighbor search, which calculates the cosine similarity of the query image’s image representation to all database image representations in the embedding space. (B) The training process of the SSL models used ResNet18 as an encoder. The image representations were passed to a projector and subjected to feature normalization. In the SimCLR method, the training loss function yields a low value when the representation of the original image (X′) and its augmentation (X′′) are close together, while it yields a larger value when X′ and a different image augmentation (Y′′) are far apart. In the TiCo method, the process still pulls X′ and X′′ close, and then the redundancy in the representation is removed using covariance contrast without using a different image (Y′′). (Image created with BioRender) Test queries The test queries were the cases available in the hospital repository after the collection of the database case was finished and were representatives of the tumor major categories in the database: odontogenic cysts, odontogenic tumors, benign and malignant tumors of the salivary gland, maxillofacial bone tumor, and soft tissue tumor. Slides that include severe artifacts that are impossible to avoid when extracting image patches were excluded. We prepared three query sets from different hospitals to test the performance: Query case set-A was collected between 2022 and 2023 from the repository of the same institution (TMDU) as the database. Only the tumor categories that have two representative cases were included as the test queries. Finally, eleven tumor categories with two cases in each were used as the test queries. Histopathologic slides were scanned to create WSIs for in-domain queries with the same device as the database image. Three selected tumor areas that are typical of the tumor type from the same slides were photographed with smartphone cameras (Samsung S21FE, iPhone 6, and Motorola g8) using an Olympus BX53 microscope with 10×, 20×, and 40× objective lens magnification to create out-of-domain-phonecam queries. The location (indoor laboratory environment) and the amount of light from the microscope were unchanged when taking the smartphone images. The query case set-B was compiled from the University of Tokyo case (approved by the IRB of The University of Tokyo No. 2019158NI). Eleven cases from eight salivary gland tumor categories were included in this study. Only benign and malignant tumors of the salivary gland could be collected for out-of-domain-B since no other major categories exist in the repository. Histopathologic slides were scanned using a NanoZoomer 2.0HT slide scanner (C9600-12, Hamamatsu Photonics, Japan) to create WSIs for out-of-domain-B queries. The query case set-C was collected from Teikyo University Hospital from 2018-2023 (approved by the IRB of Teikyo University No. 23-054). Only the tumor categories that have two representative cases were included as the test queries. Twenty cases from eight oral tumor categories were included in the study. The histopathologic slides were scanned using a NanoZoomer XR slide scanner (C12000-02; Hamamatsu Photonics, Japan) to create WSIs for out-of-domain-C queries. Patients for query cases set-B and set-C or their surrogates had the option to withdraw from this study through public disclosure according to the approved protocols. For the WSI queries, the same method was used to create image patches from the WSI as from the database image. All scanned WSIs were annotated to 3–4 representative tumor areas per WSI. From these areas, 20 image patches per magnification level (905 µm, 453 µm, and 226 µm) per slide were extracted to create image patches. The total number of image patches used for the evaluation was 2,520 images from the WSI and 594 smartphone images. The representation of each query image was calculated with each tested model. The nearest-neighbor search was performed based on cosine similarity with the database images. Examples of query images for each category in each set can be found in Figures 3 – 5 . The detailed methods for database construction, including the tumor areas selection, patch extraction, feature extraction code, and image retrieval were adapted from our previous study [ 18 ]. Download figure Open in new tab Figure 3: Examples of set-A query images from each category for each magnification level The total number of image patches extracted from WSIs is 1,320 images. Download figure Open in new tab Figure 4: Examples of set-B query images from each category for each magnification level The total number of image patches extracted from the WSIs is 660 images. Download figure Open in new tab Figure 5: Examples of set-C query images from each category for each magnification level The total number of image patches extracted from the WSIs is 1,200 images. Evaluation metrics and statistical analysis The area under the receiver operating characteristic curve (AUC) for all query images with top- k retrieved images ( k ranges from 1 to the total number of cases in the database) being the cut-points were averaged into MeanAUC. Based on the top-10 images most similar to the query, three additional metrics were evaluated. MeanAcc denotes the mean of the top-10 diagnostic accuracies (Acc) for each query. %query denotes the percentage of results that contained at least one accurate diagnosis category. The histological similarity in the retrieved results beyond diagnostic accuracy was evaluated by noting the retrieved images that did not belong to the accurate diagnosis category or any of its differential diagnoses [ 17 , 19 , 20 ] ( Table 2 ). The values are expressed as histologic inaccuracy (HI) and were averaged to determine the MeanHI. Image retrieval and all statistical analyses were conducted using Python 3.7.12 and R 4.2.2. View this table: View inline View popup Download powerpoint Table 2. List of test query categories and all their respective differential diagnoses that are represented in the database for MeanHI evaluation Model preparation and training ImageNet-1k Pre-trained CNN We tested VGG16 pre-trained on 1.2 million images from ImageNet after it was shown to perform well as an image classifier in several studies [ 22 , 23 ]. In this study, the block4_conv3 layer was used as the feature extractor because the middle layer of a convolutional neural network (CNN) architecture has been shown to capture features that are more suitable for histopathology images [ 12 ]. ImageNet-22k Pre-trained Vision Transformer We used the DINOv2 ViT-L/14 model pre-trained with the SSL method on general images from ImageNet-22k [ 26 ]. The last layer was used for feature extraction. The images were cropped to 252 pixels owing to the input size restrictions. Fine-tuned CNN We fine-tuned all the ImageNet-1k pre-trained ResNet18 models on our dataset using the supervised learning method to classify 51 categories. All the layers were trained with a learning rate of 0.001, a batch size of 32, and 100 epochs in PyTorch 1.11.0. During training, a random 90-degree rotation, random horizontal and vertical flips, color jitter, Gaussian blur, and color normalization transformation were performed. The training-to-test ratio was 8:2. CNN Trained with SSL Methods Contrastive (SimCLR) [ 21 ] and noncontrastive methods (TiCo) [ 25 ] were investigated. ResNet18 is used as the backbone. During training, random color jittering, grayscale, image scaling, horizontal and vertical flipping, 90-degree rotation, Gaussian blurring, and color augmentation were implemented ( Figure 6 ). Both models were trained with Lightly version 1.3.3, with a learning rate of 1.2, a batch size of 32 × 32 (with accumulated gradients), and 1,000 epochs. Examples of original and augmented images are shown in Figure 2B . The ResNet18 backbone model trained on 57 histopathology image datasets (38,594 image patches and 24,923 WSIs) developed by Ciga et al. 2022 was also used for comparison [ 14 ]. Download figure Open in new tab Figure 6: Data augmentation examples Random color jittering (CJ+), grayscale (GS+), image scaling (S+), horizontal and vertical flips (HF+ and VF+), 90-degree rotation (R+), and Gaussian blur (GB+), used during SSL training. Color normalization was used during model training but is not included in this visualization. The code for SSL model training is available at https://github.com/rannyrh/oralpath_CBIR . Histopathology Image-trained Vision Transformer A vision transformer-based model, Phikon, was trained with 40 million pan-cancer tiles extracted from The Cancer Genomic Atlas (TCGA) using the masked image modeling (MIM) method as an SSL framework. MIM learns meaningful representation by randomly masks portions of an image and trying to reconstruct those masked portions. This model was developed by Owkin. Inc [ 24 ] Results Model performance evaluation The highest MeanAUC for in-domain queries was achieved by ResNet18+SimCLR (0.900), followed by ResNet18+TiCo (0.897). They achieved this at the query category level for 8 out of the 11 categories. The generalizability of these models was validated using out-of-domain-phonecam queries. The highest performance for out-of-domain-phonecam queries was also achieved by ResNet18+SimCLR (0.871), followed by ResNet18+TiCo (0.857). The highest performance on the query case category levels was achieved by both SSL models for 7 out of the 11 categories. We tested the performance on queries from other institutions to further demonstrate the generalizability. A similar result was yielded by both SSL models, where ResNet18+SimCLR leads with the highest MeanAUC (0.886 for out-of-domain-B; and 0.913 for out-of-domain-D queries), followed by ResNet18+TiCo (0.881 for out-of-domain-B; and 0.905 for out-of-domain-D queries). Phikon, pre-trained on histopathological images, and DINOv2, pre-trained on large-scale general images, performed comparably well with DINOv2 leading the overall MeanAUC in out-of-domain-phonecam query, with Phikon leading in the other three query sets ( Table 3 ). View this table: View inline View popup Table 3: Mean-AUC (SD) of each test query category in in- and out-of-domain image queries. The performances of the SSL models are superior for most test query categories and the overall averages. The highest AUC for each category is marked in bold , and the second highest is marked in italics . Overall, the MeanAcc of test query set-A was highest with the SSL models: ResNet18+TiCo outperformed other models for in-domain queries (4.64), followed by ResNet18+SimCLR (4.53) with no significant difference (Wilcoxon signed-rank test with Bonferroni adjustment). The reverse was observed for the out-domain-phonecam queries (ResNet18+SimCLR (3.33) and ResNet18+TiCo (3.31)) with no significant difference (Wilcoxon signed-rank test with Bonferroni adjustment). Phikon yielded the highest MeanAcc for out-of-domain-B queries (3.79), followed by ResNet18+SimCLR (3.68). Pre-trained DINOv2 outperformed other models for out-of-domain-C queries (3.64), followed closely by ResNet18+TiCo (3.63) ( Table 4 ). The highest overall MeanAcc was consistently achieved by the SSL models at different magnification levels, except for the high-magnification in-domain queries ( Figures 7A , 7B ). The highest %query was obtained with SSL models for most query categories ( Table 5 ). Download figure Open in new tab Figure 7: MeanAcc and MeanHI comparisons (A) MeanAcc comparison for the in-domain query of all models by magnification showing the highest performance of Phikon at the highest magnification and that of ResNet18+SimCLR and ResNet18+TiCo at the moderate and lowest magnification. (B) MeanAcc comparison for out-of-domain-phonecam queries of all models by magnification shows the highest performance of ResNet18+SimCLR and ResNet18+TiCo at the highest and lowest magnification. Both model performances were comparable to that of pre-trained DINOv2 at moderate magnification. (C) MeanHI comparison for in-domain queries by magnification shows that ResNet18+SimCLR and ResNet18+TiCo outperformed other models except for the highest magnification where Phikon leads with a wider interquartile range. (D) MeanHI comparison for out-of-domain-phonecam query by magnification showing ResNet18+SimCLR and ResNet18+TiCo outperformed other models. (C-D) Please note that a lower MeanHI value denotes a higher model performance. View this table: View inline View popup Table 4. MeanAcc (SD) of each test query category in in- and out-of-domain image queries The performances of the SSL models are superior for most categories, further validating the robustness of the models under a wide range of histopathological image conditions. The highest MeanAcc for each category is marked in bold and the second highest is in italics . View this table: View inline View popup Table 5. Percentage of queries that retrieved at least one correct diagnosis among the top-10 results for each test query category (%query) in in- and out-of-domain image queries The performance of the SSL models was superior in most categories. The highest %query for each category is marked in bold , and the second highest is in italics. (%). The accuracy calculation excluded the histologic similarity between the query and retrieved images, which provides additional information about the histologic features during diagnosis. To verify whether the SSL models retrieved histologically similar images despite the low MeanAcc, MeanHI was introduced. MeanHI excludes accurate diagnosis and differential diagnosis categories, which are similar to the query and include other inaccurate categories. The lowest overall inaccuracy was consistently achieved by the SSL models, except for the high-magnification in-domain queries, indicating that these models are best at retrieving the most histologically similar images beyond accurate diagnosis ( Figures 7C , 7D , Table 6 ). The top-10 retrieved images of representative cases found by all the tested models are shown in Figures 8 and 9 . Download figure Open in new tab Figure 8: Comparison of the top-10 results of all models for out-of-domain-phonecam queries from different categories ResNet18+SimCLR and ResNet18+TiCo are consistent with the result that provides the highest Acc for different query categories. More examples are presented in Figure 9 (Green outline: accurate diagnosis category; Yellow outline: differential diagnosis categories; Red outline: inaccurate diagnosis category) Download figure Open in new tab Figure 9: Further examples of comparisons of the top 10 results of all models for out-of-domain-phonecam queries from different categories ResNet18+SimCLR and ResNet18+TiCo are consistent with the result providing the highest Acc for different query categories. The categories of retrieved images belonging to the differential diagnoses show which retrieved images have histologic similarity to the query. This comparison demonstrates the SSL model’s capability to retrieve histologically similar images when the exact accurate diagnosis is not retrieved. The pre-trained DINOv2 and Phikon also show such potential albeit less consistently across the query category than the SSL models. (Green outline: accurate diagnosis category; Yellow outline: differential diagnosis categories; Red outline: inaccurate diagnosis categories) View this table: View inline View popup Table 6. MeanHI (SD) for each test query category for in- and out-of-domain image queries The best histologic similarity was consistently retrieved by the SSL models. The histologic similarity of the results was consistently best retrieved by the SSL models. It is noteworthy that a lower MeanHI is preferable. The lowest MeanHI for each category is marked in bold and the second lowest is in italics . Discussion The diagnosis of oral pathology has long depended on histopathological image observation, which can be a burden for pathologists, especially when dealing with rare cases. In the last decade, various machine learning methods have been proposed to aid in histopathological diagnosis and improve speed and accuracy to avoid delays in diagnosis. However, the exploration for oral histopathology diagnosis has been hampered by the difficulty of obtaining an adequately extensive database that includes rare cases and constructing an effective model. To our knowledge, this is the first study to construct a large database of 30 oral tumor categories, with an additional 21 categories used as the model-training dataset. Many have argued that CBIR has greater advantages in this field. Pathologists can review CBIR results to make a final decision. However, decision bias may occur when the algorithm is unreliable. To find the best way to represent images for CBIR, we compared different methods of training the feature encoder. We then ranked the similarity of all images in the database to the test queries. The gradual concept of similarity and the multiple-ranked results of CBIR pose a challenge in interpretation. Four evaluation measures were used: MeanAUC from the whole database similarity rank; MeanAcc, %query, and MeanHI from the top-10 most similar images. MeanAUC assumes all rank cut-points are relevant to model performance in extracting histologic features, while the top-10 similar results are relevant during image observation by pathologists in future CBIR implementation. Our findings suggest that model training for feature extraction using an in-house dataset with SSL methods outperforms other popular methods in retrieving images with an accurate diagnosis and similar histology: in-domain queries in 73% of categories and out-of-domain-phonecam queries in 64% of categories ( Table 3 ). This was supported by their MeanAcc which was superior in 63% of in-domain query categories and 82% of out-of-domain-phonecam query categories ( Table 4 ). There was no significant difference in MeanAcc between the SimCLR and TiCo models for both query categories. The MeanAcc superiority of the SSL models was consistent at the in-domain low and moderate magnification levels ( Figure 7A ) and at out-of-domain-phonecam at all magnification levels ( Figure 7B ). Additionally, both SSL models retrieved fewer images without histologic similarity at all magnification levels for both query categories ( Figures 7C , 7D ), meaning when low accuracy is achieved in the result, the users get several options that are histologically similar upon CBIR implementation with the SSL models because they belong to the textbook differential diagnosis categories. From this result, users could proceed with the additional tests more easily than having to do the preliminary reference search manually. Our dataset had overlapping image patches with similar histologic features. In this situation, the SSL method was superior because it compensated for the lack of a labeled dataset for learning representations that cluster the data during training based on semantic classes in conjunction with convolutional neural networks as feature extractors, regardless of the category [ 20 ]. The most impressive performance was shown for Warthin’s tumor query, with MeanAUC values greater than 0.960 in every query set using the SSL models ( Table 3 ). Histologically, Warthin’s tumors consist of varying proportions of papillary cystic structures lined by two layers of oncocytic epithelial cells and a lymphoid stroma with germinal centers. It is one of the most common tumors of the salivary gland, especially the parotid gland, and is generally easy to diagnose microscopically owing to its characteristic pattern [ 19 ]. The SLL models were successful for most of the test query categories. Out of those categories, the MeanAcc of the SLL model for the ameloblastoma query was lower ( Table 3 ). Although ameloblastoma is one of the most common odontogenic tumors, it has diverse histologic variants: follicular, plexiform, acanthomatous, granular, basaloid, desmoplastic, or a mixture of these [ 17 ]. This diversity requires an adequate representation of each subtype in the database for greater accuracy. However, the %query indicated that the models retrieved a similar ameloblastoma type in the top-10 for more than 93% of the queries tested ( Table 5 ). With several differential diagnoses of ameloblastoma included in the database, the best MeanHI obtained for the in-domain ameloblastoma query was 5.98 by SimCLR ( Table 6 ). These categories can be considered histologically similar only if the characteristics of certain subtypes are captured. For example, islands of odontogenic epithelium with ameloblastic features in the follicular type may resemble ameloblastic fibroma [ 19 ]. Updating the database with newly encountered subtypes continuously would improve the accuracy of rare tumor subtypes. Although CBIR works by retrieving similar images that can be considered a digital second opinion, the result may contain images from different categories, with many having similar or indistinguishable histology. Arguably, the range of MeanAcc values obtained with the SSL models, 1.00 to 8.94 ( Tables 3 and 4 ), is considerably wide. However, 55% to 100% of the total queries retrieved at least one of 10 images from the correct category ( Table 5 ), and 6.46 to 1.12 out of the 10 images had no histologic similarity to the query image ( Table 6 ). This implies that displaying the complete top-10 results, including the correct diagnosis and differential diagnosis, as shown in Figures 8 and 9 , could be significant for pathologists to narrow the differential diagnoses and conduct further research efficiently. To further improve usability, it is necessary to include clinical and other findings, such as the location of the tumor, patient history, and diagnostic criteria, which are usually essential to making a diagnosis by pathologists, when developing a CBIR system, especially in the oral region where tissue types are diverse. This study implements patch-based CBIR. Some CBIR systems can analyze WSIs of which implementation is prospective in developed countries. As expensive WSI scanners are not universally installed in oral laboratories, the image-capturing equipment accessible to pathologists differs considerably across regions. Microscope images captured directly using a smartphone camera could be the easiest mode for education, image sharing, and case consultation [ 27 , 28 ]. By using patch-based CBIR where pathologists only need to select the tumor areas and capture them with smartphone cameras to create input, this technology is more accessible globally. Variations in image color and resolution resulting from these differences hinder obtaining reliable results. We tested the robustness of each model to domain shifts by testing the models on out-of-domain queries using WSIs from multiple institutions captured by different scanners and smartphone cameras. SSL models performed best for most query categories, with SimCLR or TiCo achieving the best MeanAUC for over 68% of out-of-domain query categories, from 0.839 to 0.999 ( Table 3 ), confirming the previous finding that SSL is more robust to domain shifts than supervised learning in some datasets, including pathological images [ 29 ]. Interestingly, the performances of the vision transformer models (pre-trained DINOv2 and Phikon) always come second best to the SSL models in MeanAUC and are comparable in MeanAcc to that of the SSL models on out-of-domain query sets ( Tables 3 and 4 ). Although further investigation is needed, this result may be considered when choosing methods for a CBIR system. If the system is designed for an in-house database and query or a scenario applicable in large hospitals, SSL models trained on in-house cases are the optimal choice. However, where the system is designed to handle out-of-domain queries, using a pre-trained vision transformer model becomes a viable alternative to eliminate the need to train the SSL model, which could be computationally expensive. Study limitations Limitations of this study include that the experiments involved a single query to retrieve similar images. An algorithm that supports more information in the query, such as multiple query algorithms and filters for location or other diagnostic criteria, would improve retrieval accuracy and provide better support for diagnosis. Our study is limited to test queries from the same geographical area as the SSL model-training dataset. Collecting query cases from a more diverse area would be beneficial in future CBIR development to further challenge the generalizability of the result. The comparative methodology did not emphasize histopathology characteristics that differentiate between benign or malignant tumors, such as capsule invasion and mitotic activity in basal cell adenoma vs. basal cell adenocarcinoma but focused on how such image retrieval tools would be beneficial in reducing to several differential diagnoses and recalling diagnosis criteria before following up with ancillary tests if necessary. Image retrieval is less likely to mislead decision-makers owing to model overfit than a conventional classification method that predicts the possible tumor diagnosis. Nonetheless, a sequel of observations would still be needed when image retrieval is utilized. This study provides insights as the first step to developing a CBIR algorithm by observing retrieval accuracy with strict tumor category criteria on a relatively small database and did not investigate the impact of the result on decision-making in clinical settings. The implementation of CBIR as a well-rounded system to be incorporated into the comprehensive diagnostic process is beyond the scope of this study and observation of the interaction between pathologists and a CBIR system for common and rare diagnoses is needed before the system is used in clinical settings. Conclusion This study highlighted various methods to develop an effective CBIR model and presented key measures to determine the best approach for future clinical usage. We have shown that using SSL methods for deep neural network training is an effective way to develop a CBIR system for histopathological diagnosis of oral tumors compared to other commonly used methods. Vision transformer models, though slightly less effective than SSL models, still provided strong performance and could be a viable alternative for out-of-domain queries. These approaches have considerable potential to create a clinically useful image retrieval system that accelerates the diagnostic process and improves accuracy. Data Availability All data produced in the present study are available upon reasonable request to the authors. Author Contributions R. R. Herdiantoputri: Contributed to the conception, design, data acquisition, drafting, and critical revision of the manuscript. D. K. contributed to the conception, design, drafting, and critical revision of the manuscript. M. Ochi: Contributed to the design and critical revision of the manuscript. Y. Fukawa: Contributed to data acquisition and critically revised the manuscript. K. Kayamori: Contributed to data acquisition, and critically revised the manuscript., M. Tsuchiya: Contributed to data acquisition, and critically revised the manuscript. Y. Kikuchi: Contributed to data acquisition and critically revised the manuscript. T. Ushiku: Contributed to the data acquisition and critically revised the manuscript. T. Ikeda: Contributed to the data acquisition and critically revised the manuscript. Ishikawa: Contributed to the conception, design, drafting, and critical revision of the manuscript. All authors provided their final approval and agreed to be accountable for all aspects of this work. Declaration of Conflicting Interests The authors declare no potential conflicts of interest concerning the research, authorship, or publication of this article. Funding This study was supported by AMED Practical Research for Innovative Cancer Control under grant number JP 23ck0106640 to S.I. and the JSPS KAKENHI Grant-in-Aid for Scientific Research (B) under grant number 21H03836 to D.K. Acknowledgments We thank Editage ( www.editage.jp ) for the English language review. We thank the pathologists who contributed to the image verification. References 1. ↵ Kiehl T-R : Digital and Computational Pathology: A Specialty Reimagined . In: Ehsani S , Glauner P , Plugmann P , Thieringer FM , eds. The Future Circle of Healthcare . Springer International Publishing, Cham ; 2022 . doi: 10.1007/978-3-030-99838-7_12 OpenUrl CrossRef 2. ↵ Tommasino C , Merolla F , Russo C , Staibano S , Rinaldi AM : Histopathological Image Deep Feature Representation for CBIR in Smart PACS: J Digit Imaging . 2023 , 36 : 2194 – 209 . doi: 10.1007/s10278-023-00832-x OpenUrl CrossRef 3. ↵ Rao RS , Shivanna DB , Lakshminarayana S , et al : Ensemble Deep-Learning-Based Prognostic and Prediction for Recurrence of Sporadic Odontogenic Keratocysts on Hematoxylin and Eosin Stained Pathological Images of Incisional Biopsies . J Pers Med . 2022 , 12 : 1220 . doi: 10.3390/jpm12081220 OpenUrl CrossRef 4. ↵ Giraldo-Roldan D , Ribeiro ECC , Araújo ALD , et al : Deep learning applied to the histopathological diagnosis of ameloblastomas and ameloblastic carcinomas . J Oral Pathology Medicine . 2023 , 13481 : doi: 10.1111/jop.13481 OpenUrl CrossRef 5. ↵ Kumar N , Gupta R , Gupta S : Whole Slide Imaging (WSI) in Pathology: Current Perspectives and Future Directions . J Digit Imaging . 2020 , 33 : 1034 – 40 . doi: 10.1007/s10278-020-00351-z OpenUrl CrossRef 6. ↵ Van Der Laak J , Litjens G , Ciompi F : Deep learning in histopathology: the path to the clinic . Nat Med . 2021 , 27 : 775 – 84 . doi: 10.1038/s41591-021-01343-4 OpenUrl CrossRef PubMed 7. ↵ Hameed IM , Abdulhussain SH , Mahmmod BM : Content-based image retrieval: A review of recent trends . Cogent Engineering . 2021 , 8 : 1927469 . doi: 10.1080/23311916.2021.1927469 OpenUrl CrossRef 8. ↵ Dehkharghanian T , Bidgoli AA , Riasatian A , et al : Biased data, biased AI: deep networks predict the acquisition site of TCGA images . Diagn Pathol . 2023 , 18 : 67 . doi: 10.1186/s13000-023-01355-3 OpenUrl CrossRef 9. ↵ Springenberg M , Frommholz A , Wenzel M , Weicken E , Ma J , Strodthoff N : From modern CNNs to vision transformers: Assessing the performance, robustness, and classification strategies of deep learning models in histopathology . Medical Image Analysis . 2023 , 87 : 102809 . doi: 10.1016/j.media.2023.102809 OpenUrl CrossRef 10. Kalra S , Tizhoosh HR , Choi C , Shah S , Diamandis P , Campbell CJV , Pantanowitz L : Yottixel - An Image Search Engine for Large Archives of Histopathology Whole Slide Images . Medical Image Analysis . 2020 , 65 : 101757 . doi: 10.1016/j.media.2020.101757 OpenUrl CrossRef 11. Albashish D : Ensemble of adapted convolutional neural networks (CNN) methods for classifying colon histopathological images . PeerJ Computer Science . 2022 , 8 : 1031 . doi: 10.7717/peerj-cs.1031 OpenUrl CrossRef 12. ↵ Komura D , Kawabe A , Fukuta K , et al : Universal encoding of pan-cancer histology by deep texture representations . Cell Reports . 2022 , 38 : 110424 . doi: 10.1016/j.celrep.2022.110424 OpenUrl CrossRef 13. ↵ Wang X , Yang S , Zhang J , et al : Transformer-based unsupervised contrastive learning for histopathological image classification . Medical Image Analysis . 2022 , 81 : 102559 . doi: 10.1016/j.media.2022.102559 OpenUrl CrossRef PubMed 14. ↵ Ciga O , Xu T , Martel AL : Self supervised contrastive learning for digital histopathology . Machine Learning with Applications. Machine Learning with Applications . 2022 , 7 : 100198 . doi: 10.1016/j.mlwa.2021.100198 OpenUrl CrossRef 15. Shmatko A , Ghaffari Laleh N , Gerstung M , Kather JN : Artificial intelligence in histopathology: enhancing cancer research and clinical oncology . Nat Cancer . 2022 , 3 : 1026 – 38 . doi: 10.1038/s43018-022-00436-4 OpenUrl CrossRef 16. ↵ Ben-Shaul I , Shwartz-Ziv R , Galanti T , Dekel S : Reverse Engineering Self-Supervised Learning . arXiv . 2023 , doi: 10.48550/ARXIV.2305.15614 OpenUrl CrossRef 17. ↵ Organisation mondiale de la santé, Centre international de recherche sur le cancer: WHO classification of head and neck tumours , 4th Edition, Volume 9 . IARC, Lyon ; 2017 . 18. ↵ Herdiantoputri RR , Komura D , Fujisaka K , Ikeda T , Ishikawa S : Deep texture representation analysis for histopathological images . STAR Protocols. STAR Protocols . 2023 , 4 : 102161 . doi: 10.1016/j.xpro.2023.102161 OpenUrl CrossRef 19. ↵ Gnepp DR , Bishop JA : Gnepp’s Diagnostic Surgical Pathology of the Head and Neck . 3 . Edition. Elsevier, Philadelphia ; 2020 . 20. ↵ Bishop JA , Thompson LDR , Wakely PE , Weinreb I : Tumors of the salivary glands. American Registry of Pathology, Arlington , VA ; 2021 . 21. ↵ Chen T , Kornblith S , Norouzi M , Hinton G : A Simple Framework for Contrastive Learning of Visual Representations . arXiv . 2020 , doi: 10.48550/arXiv.2002.05709 OpenUrl CrossRef 22. ↵ Liu M , Yi M , Wu M , Wang J , He Y : Breast Pathological Image Classification Based on VGG16 Feature Concatenation . J Shanghai Jiaotong Univ (Sci . 2022 , 27 : 473 – 84 . doi: 10.1007/s12204-021-2398-x OpenUrl CrossRef 23. ↵ Zhou P , Cao Y , Li M , et al : HCCANet: histopathological image grading of colorectal cancer using CNN based on multichannel fusion attention mechanism . Sci Rep . 2022 , 12 : 15103 . doi: 10.1038/s41598-022-18879-1 OpenUrl CrossRef 24. ↵ Filiot A , Ghermi R , Olivier A , et al : Scaling Self-Supervised Learning for Histopathology with Masked Image Modeling . medRxiv . 2023 , doi: 10.1101/2023.07.21.23292757 OpenUrl Abstract / FREE Full Text 25. ↵ Zhu J , Moraes RM , Karakulak S , Sobol V , Canziani A , LeCun Y : TiCo: Transformation Invariance and Covariance Contrast for Self-Supervised Visual Representation Learning . arXiv . 2022 , doi: 10.48550/ARXIV.2206.10698 OpenUrl CrossRef 26. ↵ Oquab M , Darcet T , Moutakanni T , et al : DINOv2: Learning Robust Visual Features without Supervision . arXiv . 2023 , doi: 10.48550/arXiv.2304.07193 OpenUrl CrossRef 27. ↵ Morrison AO , Gardner JM : Microscopic Image Photography Techniques of the Past, Present, and Future . Archives of Pathology & Laboratory Medicine . 2016 , 140 : 618 – 618 . doi: 10.5858/arpa.2016-0903-CO OpenUrl CrossRef 28. ↵ Maekawa Y , Tanahashi N , Yoneda M , Morishita Y : Smartphone-based imaging technology for histopathology image education in lieu of a tabletop microscope . Japanese Journal of Medical technology . 2023 , 72 : 614 – 618 . doi: 10.14932/jamt.23-31 OpenUrl CrossRef 29. ↵ Shi Y , Torr PHS , Daunhawer I , Vogt JE , Sanyal A : How Robust is Unsupervised Representation Learning to Distribution Shift . arXiv . 2022 , doi: 10.48550/arXiv.2206.08871 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted May 31, 2024. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology Ranny Rahaningrum Herdiantoputri , Daisuke Komura , Mieko Ochi , Yuki Fukawa , Kou Kayamori , Maiko Tsuchiya , Yoshinao Kikuchi , Tetsuo Ushiku , Tohru Ikeda , Shumpei Ishikawa medRxiv 2024.05.30.24308181; doi: https://doi.org/10.1101/2024.05.30.24308181 Share This Article: Copy Citation Tools Benchmarking Deep Learning-based Image Retrieval of Oral Tumor Histology Ranny Rahaningrum Herdiantoputri , Daisuke Komura , Mieko Ochi , Yuki Fukawa , Kou Kayamori , Maiko Tsuchiya , Yoshinao Kikuchi , Tetsuo Ushiku , Tohru Ikeda , Shumpei Ishikawa medRxiv 2024.05.30.24308181; doi: https://doi.org/10.1101/2024.05.30.24308181 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Pathology Subject Areas All Articles Addiction Medicine (574) Allergy and Immunology (865) Anesthesia (304) Cardiovascular Medicine (4458) Dentistry and Oral Medicine (445) Dermatology (383) Emergency Medicine (611) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1517) Epidemiology (15249) Forensic Medicine (30) Gastroenterology (1132) Genetic and Genomic Medicine (6621) Geriatric Medicine (669) Health Economics (1002) Health Informatics (4560) Health Policy (1372) Health Systems and Quality Improvement (1616) Hematology (544) HIV/AIDS (1272) Infectious Diseases (except HIV/AIDS) (15936) Intensive Care and Critical Care Medicine (1107) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6639) Nursing (346) Nutrition (1000) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3349) Ophthalmology (981) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5464) Public and Global Health (9257) Radiology and Imaging (2211) Rehabilitation Medicine and Physical Therapy (1372) Respiratory Medicine (1198) Rheumatology (598) Sexual and Reproductive Health (716) Sports Medicine (533) Surgery (715) Toxicology (100) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a038360d9dc5df88',t:'MTc4MDA4NjE0Ng=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();