BioFuse: An embedding fusion framework for biomedical foundation models

BioFuse: An embedding fusion framework for biomedical foundation models | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results BioFuse: An embedding fusion framework for biomedical foundation models View ORCID Profile Mirza Nasir Hossain , David Harris-Birtill doi: https://doi.org/10.1101/2025.03.01.640976 Mirza Nasir Hossain 1 University of St Andrews , St Andrews, Fife, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mirza Nasir Hossain For correspondence: mnh3{at}st-andrews.ac.uk David Harris-Birtill 1 University of St Andrews , St Andrews, Fife, United Kingdom Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Preview PDF Abstract The biomedical field has witnessed a surge in pre-trained foundation mod-els excelling in specific sub-domains such as radiology and histopathology. While integrating these models promises a more comprehensive understand-ing of biomedical data, it poses challenges in model compatibility and feature fusion. We present BioFuse, a novel open-source framework designed to generate optimised biomedical embeddings. BioFuse utilises a pool of 9 state-of-the-art foundation models to create task-specific embeddings. It employs grid search to automatically identify the optimal combination of models, fusing their embeddings through vector concatenation. On the MedMNIST+ benchmark, using XGBoost as the downstream classifier, Bio-Fuse outperforms several existing methods, achieving SOTA AUC in 5/12 datasets while maintaining near-SOTA performance across most remain-ing datasets. Remarkably, our experiments reveal unexpected cross-modal capabilities, with histopathology and radiology models showing strong performance when applied to other imaging modalities. BioFuse features a high-level API 1 for immediate deployment and an extensible architecture to incorporate future models and fusion techniques. We anticipate BioFuse will not only enhance the utility of foundation models in biomedicine but also open new avenues for uncovering cross-modal relationships in biomedical data. Introduction Artificial Intelligence (AI) is driving advances in biomedical research and clinical practice [ 1 – 3 ]. Foundation models — large, pre-trained models capable of performing a wide variety of tasks with minimal adaptation — have emerged as key tools in this domain. Trained on vast, diverse datasets [ 4 ], these models have demonstrated promising performance across various biomedical sub-domains [ 5 ]. In medical imaging, foundation models are utilised for detecting and classifying abnormalities in radiographs [ 5 ], generating diagnostic reports [ 6 ], segmenting images to delineate organs and tumours [ 7 ], analysing histopathology slides for disease diagnosis [ 8 , 9 ], and integrating information across imaging modalities to enhance diagnostic accuracy [ 10 ]. In clinical natural language processing (NLP), they are employed for clinical text summarization [ 11 ] and information extraction [ 12 ], while in genomics, they assist in deciphering the language of non-coding DNA [ 13 ] and enable accurate cell type annotation [ 14 ]. Typically, researchers manually select foundation models tailored for specific tasks [ 15 ]. These models increasingly serve as feature extractors, transforming input data into dense numerical representations called embeddings [ 16 ]. These embeddings encapsulate complex features of the input data in high-dimensional vectors, allowing for efficient computation and comparison. However, foundation models are often trained on single modalities, with research groups developing specialized architectures for specific domains [ 17 – 19 ]. While this approach enables exceptional performance within designed scopes, these models may have learned features relevant beyond their original domains. As domain-specific models become increasingly refined [ 20 ], there’s a growing need to explore their broader applicability across biomedical tasks and modalities. Combining embeddings from multiple models allows us to exploit modalityspecific encodings while gaining new perspectives through cross-modal integration. Various approaches have demonstrated the value of leveraging multiple modalities for instance, ConVIRT [ 21 ] utilises contrastive learning between paired images and text to enhance chest X-ray classification performance. Such cross-modal techniques suggest that strategic integration of specialized models could generate more comprehensive representations of biomedical data and unlock insights inaccessible to single-modality approaches. This leads us to ask two key questions: 1. Does combining the embeddings of multiple foundation models trained on different biomedical modalities improve task performance, as measured by AUC (area under the curve) and accuracy? 2. How effectively do foundation models transfer knowledge across different biomedical imaging modalities, as evidenced by comparative AUC and accuracy scores in cross-modal applications? Despite the potential benefits, effectively integrating multiple foundation models presents major challenges. Retraining models on combined data from multiple modalities demands extensive computational resources and expertise [ 22 ]. While combining pre-trained model embeddings offers an alternative [ 23 , 24 ], the rapidly growing number of foundation models [ 25 ] makes identifying optimal combinations increasingly complex. Moreover, the lack of standardized frameworks for embedding fusion hinders our ability to fully exploit these models’ collective knowledge. Our Contributions In response to these challenges, we introduce BioFuse, an open-source framework designed to harness the collective power of diverse biomedical foundation models. This paper makes the following key contributions: 1. Enhanced Performance on the MedMNIST+ Benchmark: Using embeddings generated by BioFuse combined with XGBoost classification, we outperform several existing baselines on the MedMNIST+ benchmark, achieving the highest test AUC in 5 of 12 datasets and best accuracy in 2 datasets. The framework’s embeddings maintain near-SOTA performance (within 2% margin) across most remaining datasets, demonstrating the effectiveness of our embedding fusion approach. 2. Revealing Cross-Modal Capabilities in Biomedical Foundation Models: Our experiments reveal unexpected cross-modal transfer in both histopathology and radiology models, showcasing the potential for models trained on one imaging modality to generate effective embeddings for others. 3. An Extensible Framework for Optimised Biomedical Embedding Generation: BioFuse provides a high-level API that simplifies the generation of task-specific embeddings. It automatically identifies optimal combinations of foundation models and fuses their outputs, addressing the challenges of model integration. The framework’s architecture allows for easy incorporation of new models and fusion techniques, ensuring adaptability to emerging developments in biomedical AI. By providing a standardized approach to embedding fusion and automating model selection, BioFuse enhances the utility of foundation models in biomedicine while opening new avenues for uncovering relationships between diverse biomedical data representations. Background Foundation Models in Biomedicine Foundation models, driven by Large Language Models (LLMs), have ushered in a new era in deep learning [ 26 ]. Three key factors enabled their development: the Transformer architecture [ 27 ] for processing sequential data, advances in GPU capabilities [ 28 ], and extensive training datasets [ 29 , 30 ]. These billion-parameter models [ 12 ] require substantial computing infrastructure for training [ 31 ], but can then solve tasks beyond their original training objectives [ 32 ]. While they support zero-shot inference and finetuning [ 26 ], they are commonly used as feature extractors for classification models [ 16 , 33 , 34 ]. Foundation models are typically trained using self-supervised learning (SSL) techniques, including Contrastive Learning [ 35 ], Masked Image Modelling (MIM) [ 36 ], and self-distillation [ 37 ]. These methods create proxy tasks from unlabelled data: contrastive learning differentiates between similar and dissimilar samples, MIM reconstructs masked image regions, and self-distillation leverages the model’s own predictions as training targets. By utilizing the inherent structure of data, SSL enables training on much larger datasets without manual annotation. This approach has demonstrated superior performance and transferability compared to traditional supervised learning [ 35 , 36 , 38 ], making it particularly valuable in medical imaging where labelled data is scarce. Unimodal Foundation Models Biomedical imaging foundation models have demonstrated remarkable effectiveness when trained on single modalities (unimodal training). In computational pathology, models like UNI [ 39 ], UNI2 [ 40 ], Prov-GigaPath [ 41 ], and Hibou-B [ 42 ] have set new benchmarks through increasingly larger-scale training. UNI outperforms prior models across 34 clinical tasks, while its successor UNI2 expands capabilities to both H&E and Immunohistochemistry (IHC). Prov-GigaPath achieves state-of-the-art performance in cancer subtyping and mutation prediction, and Hibou-B demonstrates high adaptability across pathology applications. In radiology, RAD-DINO [ 43 ], pre-trained on a vast corpus of chest X-rays, excels in detecting conditions and capturing detailed features crucial for biomarker discovery and prognosis prediction. Built on architectures like Vision Transformers (ViT) and trained with self-supervised learning, these models demonstrate how large-scale training captures modality-specific nuances and enables superior biomedical imaging performance. Vision Language Models Vision-language models (VLMs) integrate visual and textual data to learn joint representations for tasks like image classification, retrieval, and captioning. BioMedCLIP [ 44 ] and PubMedCLIP [ 45 ] achieve state-of-the-art performance across various biomedical benchmarks, while CheXagent [ 46 ] excels in chest X-ray interpretation tasks and CONCH [ 47 ] demonstrates superior performance across 13 histopathology benchmarks. These models demonstrate how integrating visual and textual information enhances biomedical imaging analysis. Limitations of Foundation Models Despite their capabilities, biomedical foundation models face serious limitations including restricted cross-domain applicability, high computational demands, and challenges in interpretability crucial for clinical decisionmaking. While using them as feature extractors mitigates the risk of hallucinations [ 20 ], effectively leveraging the collective knowledge of multiple specialized models remains an open challenge that could enhance performance and provide more robust biomedical insights. Related Work Prior work relevant to BioFuse spans three key areas: embedding fusion approaches that combine features from multiple models, cross-modal methods that transfer knowledge between domains, and automated selection techniques that optimize model combinations. Embedding Fusion Recent works have demonstrated the benefits of combining embeddings from multiple models. In histopathology, Neidlinger et al. [ 24 ] showed that combining four foundation models improved tumor classification and survival prediction across 13 patient cohorts, but their approach was limited to a single modality and lacked a framework for optimal model selection. Similarly, Zarif et al. [ 48 ] and Dong et al. [ 49 ] combined two and four pre-trained CNNs respectively for breast and liver cancer classification, but their approaches used bespoke architectures specific to their tasks. In clinical NLP, BioFLAIR [ 50 ] combines two embedding models (FLAIR and BioELMo) for biomedical named entity recognition, while the Concatenated BioMed-Transformers [ 51 ] fuses three transformer models for medical text classification, but both were limited to single modalities. Cross-Modal Transfer Cross-modal approaches have attempted to bridge different modalities in biomedical data. Zipper [ 52 ] combines two pre-trained unimodal models for speech recognition and text-to-speech tasks using multi-tower decoders with cross-attention, but its computational complexity limits practical application. BioBridge [ 53 ] uses knowledge graphs to connect two unimodal foundation models for protein sequence-text retrieval and drug design tasks, yet fails to fully exploit the embedded knowledge across different biomedical domains. Automated Model Selection In automated model selection, ACE [ 54 ] uses reinforcement learning to select optimal combinations of word embedding models for NLP tasks like named entity recognition and part-of-speech tagging. However, its focus on text-only tasks fails to address the multimodal nature of biomedical data, and its computational demands make it impractical for high-dimensional data where rapid model selection is often needed. Limitations of Related Work While these approaches have made impressive strides, they often focus on specific domains or tasks, lacking the versatility to leverage image extraction capabilities across models trained on different modalities. Moreover, the untapped potential in combining foundation models to enhance performance and generalisability across diverse modalities remains a non-trivial challenge. BioFuse aims to address these limitations by providing a flexible framework that can integrate multiple foundation models across various biomedical domains, facilitating both embedding fusion and cross-modal transfer while automating the selection of optimal model combinations. BioFuse Overview BioFuse is an open-source framework that automates the selection, extraction, and fusion of embeddings from multiple pre-trained biomedical foundation models across diverse modalities such as radiographs, histopathology slides, and clinical text. Its modular design supports seamless integration of new models and fusion methods, keeping pace with rapid advancements. BioFuse’s primary goal is to generate optimal embeddings by leveraging multiple models via vector concatenation. These fused embeddings encapsulate multimodal information in a unified format. To assess their quality, BioFuse employs an approach similar to linear probing [ 55 ], but with a more sophisticated classifier. Specifically, we use XGBoost [ 56 ], a powerful tree-based machine learning algorithm, to train on the frozen, fused embeddings for various downstream tasks. This evaluation method provides a reliable measure of embedding quality without requiring computationally expensive fine-tuning of the foundation models themselves. Users can utilise these embeddings as input features for custom models tailored to specific research questions or clinical applications. By offering optimised embeddings and demonstrating their effectiveness through advanced probing, BioFuse serves as a versatile feature extraction framework for biomedical imaging tasks. System Architecture BioFuse’s architecture comprises three components: pre-trained foundation models, the BioFuseModel, and the search module (see Fig 1 ): 1. Pre-trained Foundation Models: Existing, pre-trained models from various biomedical domains are loaded without fine-tuning to preserve their original capabilities. 2. BioFuseModel: The core component that processes and integrates embeddings from multiple pre-trained models in a single forward pass per input. It preprocesses each model’s input, extracts embeddings, and fuses them via vector concatenation, preserving the strengths of each model while ensuring efficiency. 3. Search Module: Evaluates embedding performance from different model combinations (each represented by a BioFuseModel) by computing validation accuracy using XGBoost as a lightweight downstream classifier. This step automates identifying optimal configurations. Download figure Open in new tab Fig 1. BioFuse architecture and workflow. The upper section illustrates the training process: input samples are preprocessed and fed into multiple foundation models to generate embeddings, which are then concatenated and used to train a classifier. The lower section shows the evaluation process using the same BioFuseModel architecture on a validation set, followed by performance assessment of the trained model. Foundation Models in BioFuse Foundation models in BioFuse were selected based on architectural diversity, performance, efficiency, and accessibility. Both unimodal and vision-language models (VLMs) are included to capture diverse biomedical information, with VLMs chosen for their pre-training on paired text-image data. Models trained on high-quality biomedical datasets in various imaging modalities, from macroscopic radiological images to microscopic histological data, were preferred for a comprehensive representation of knowledge. The selection process prioritized models documented in peer-reviewed literature with demonstrated excellence across multiple biomedical applications. We carefully balanced computational efficiency against performance to ensure BioFuse remains practical for real-world implementation. Additionally, we favored models with standardized implementations on platforms such as Hugging Face [ 57 ] to enhance accessibility and community support. Through this thoughtful curation, BioFuse incorporates a diverse array of foundation models that collectively provide robust and adaptable capabilities across the biomedical domain. Table 1 summarizes the foundation models supported by BioFuse, including dataset size, pre-training methods, and encoder types. View this table: View inline View popup Download powerpoint Table 1. Summary of Foundation Models supported in BioFuse Embedding Extraction BioFuse streamlines the extraction of embeddings from diverse pre-trained foundation models across multiple biomedical imaging modalities. The framework leverages established libraries including Hugging Face Transformers [ 59 ], OpenCLIP [ 60 ], and PyTorch Image Models [ 61 ] to facilitate this process. The system automatically handles model and preprocessor loading while applying appropriate model-specific preprocessing—such as image resizing and normalization—before generating embeddings. The framework employs GPU acceleration to enhance inference speed when processing large-scale datasets. Memory management is optimised through immediate release of intermediate outputs after use, enabling the system to scale effectively with increasingly complex inputs. These technical enhancements ensure BioFuse can efficiently handle diverse biomedical imaging modalities while maintaining computational efficiency in resourceconstrained environments. Fusion Methodology BioFuse fuses embeddings from multiple pre-trained models using vector concatenation , combining embeddings into a single representation. Let be the embedding from the i -th model, where d i is its dimensionality. Given n models, the fused embedding is: where [;] denotes concatenation along the feature dimension, yielding a total size: For instance, if then . Advantages of Vector Concatenation Efficient & Simple : No additional parameters or projection layers, reducing computational complexity. Feature Preservation : Retains all modality-specific features without loss. Scalable : New models can be seamlessly integrated without retraining fusion components. Interpretable : Maintains per-model feature separation, aiding downstream classifiers. Automated Selection of Models BioFuse automates model selection, identifying the optimal set for each task without manual intervention. It evaluates all 2 n − 1 model combinations, leveraging GPU-accelerated XGBoost to efficiently search this exponential space. Embeddings are fused and rapidly assessed using gradient boosting. GPU acceleration mitigates computational overhead by parallelizing tree construction and handling large feature matrices efficiently. This enables BioFuse to evaluate model combinations in seconds and process larger datasets within minutes, ensuring thorough exploration of the combinatorial space. An on-disk cache stores previously generated embeddings to avoid redundant computations. BioFuse selects the combination with the highest validation accuracy, ensuring embeddings are tailored to the specific task while maintaining generalisation within the dataset. Experimental Setup Dataset As Illustrated in Fig 2 , our study uses 12 2D datasets from the MedMNIST+ benchmark [ 62 ], a large-scale, MNIST-like collection of standardized biomedical images designed for various machine learning tasks in the medical domain. Key features of the dataset include: 1. Multi-modal: MedMNIST+ covers a wide range of biomedical imaging modalities, including X-Ray, OCT, Ultrasound, CT, Histopathology and Electron Microscopy images. 2. Standardized: All images are pre-processed into a uniform 224×224 resolution for 2D datasets, with corresponding classification labels. The datasets also come with consistent train-validation-test splits. 3. Multi-task: The dataset supports various machine learning tasks, including binary and multi-class classification, ordinal regression, and multi-label classification. 4. Multi-scale: MedMNIST includes approximately 708K 2D images, with dataset sizes ranging from 780 to 236,386 samples. Download figure Open in new tab Fig 2. Visual overview of the MedMNIST+ 2D datasets. Sample images at 224×224 resolution showcase the diverse collection spanning multiple medical imaging modalities, illustrating the wide range of diagnostic tasks represented in the benchmark. The decision to focus on 2D datasets was driven by the use of a simple classifier which is well-suited for 2D image analysis tasks. Table 2 provides an overview of the MedMNIST2D datasets used in this study. View this table: View inline View popup Download powerpoint Table 2. Overview of MedMNIST+ 2D Datasets MedMNIST+ enables a comprehensive evaluation of BioFuse across multi-class, binary, and multi-label classification, spanning diverse dataset sizes and class distributions. For consistency, we treat the ordinal regression task (RetinaMNIST) as a multi-class classification problem, a standard approach for handling ordinal data in machine learning [ 72 ]. Experimental Design Objectives Our experiments assess: 1. Performance Improvement: Evaluate whether fusing embeddings via vector concatenation outperforms individual models on MedMNIST+ using AUC and Accuracy. 2. Cross-modal Transfer: Test if features from one modality (e.g., histopathology) enhance performance when combined with another (e.g., radiographs), measured by AUC and Accuracy. 3. Generalisation: Assess BioFuse’s ability to maintain strong performance across diverse biomedical domains using nine foundation models. These objectives establish BioFuse as a versatile framework for multimodal biomedical imaging, leveraging complementary information to improve classification performance across datasets. Hardware Configuration The experiments were conducted on a server with an NVIDIA A100 GPU (80GB VRAM), an AMD EPYC 7713 CPU (64-core, 128-thread CPU), and 1TB RAM. While the CPU and memory were shared, the GPU was dedicated to BioFuse, ensuring uninterrupted computation for embedding extraction, model fusion, and classification. Experimental Procedure XGBoost For internal evaluation (see Fig 1 ), we used XGBoost [ 56 ], a scalable gradient-boosted decision tree algorithm known for its efficiency in binary and multi-class classification. The model was trained on fused embeddings from multiple foundation models. Although embedding order can influence performance, we found this effect negligible. Optimizing order introduces substantial complexity—2 n − 1 model combinations and n ! possible orders per combination—making exhaustive search impractical. To maintain computational tractability, we fixed the concatenation order. Hyperparameters were manually selected following [ 73 ], balancing efficiency and accuracy. We set n_estimators = 250, learning_rate = 0.1, and max_depth = 6 to prevent overfitting while capturing biomedical data patterns. The objective function was multi:softprob for multi-class, binary:logistic for binary classification, and a one-versus-rest strategy for multi-label tasks. Training used XGBoost’s GPU acceleration to reduce computation time. Training and Evaluation Workflow Our evaluation of BioFuse follows a structured workflow, as illustrated in Fig 3 : 1. Dataset Preparation: BioFuse is provided with training and validation sets for each dataset in MedMNIST+ 2D. 2. Model Selection: It identifies the optimal combination of foundation models for each dataset (see Fig 1 ). 3. Embedding Generation: Using the selected model combination, BioFuse generates training and validation embeddings and returns a configured BioFuseModel (embedding generator). 4. Classifier Selection and Tuning: For the final evaluation, we use XGBoost, tuning its hyperparameters on training embeddings and selecting the configuration that achieves the highest validation accuracy. 5. Test Embedding Generation: The trained BioFuseModel generates embeddings for the test set, ensuring consistency with the training and validation process. 6. Final Evaluation: The tuned XGBoost model (from Step 4) is evaluated on the test set embeddings to assess overall performance. Download figure Open in new tab Fig 3. High-level workflow of BioFuse within a typical machine learning pipeline. BioFuse serves as a sophisticated embedding generator, accepting training and validation sets as inputs. As outputs, BioFuse provides embeddings for both the training and validation sets, along with a configured BioFuseModel. This BioFuseModel acts as an embedding generator for subsequent use on the test set. To ensure a comprehensive final evaluation on the test sets, we conducted independent hyperparameter tuning for each dataset in the MedMNIST+ benchmark (as described in Step 4). Details on the specific hyperparameters are provided in Appendix B ( Table 6 )for reproducibility. Performance was measured using test AUC and accuracy, with results averaged over three independent runs to enhance robustness. Evaluation Metrics We employ two complementary metrics in our evaluation framework: Accuracy is used for internal model selection during BioFuse’s automated search process: Area Under the ROC Curve (AUC-ROC) serves as our primary reporting metric for final test set performance. This choice aligns with MedMNIST+ benchmarks and provides a threshold-independent assessment of model quality. AUC-ROC represents the probability that a randomly chosen positive example ranks higher than a negative one [ 74 ], with values ranging from 0.5 (random performance) to 1.0 (perfect classification). For comprehensive comparison, we report both AUC-ROC and accuracy in our final results tables. Results Overview On the MedMNIST+ benchmark, BioFuse outperforms several existing methods, achieving the highest test AUC on 5 of 12 datasets and near-SOTA performance on six more. It also achieves the best test accuracy on 2 of 12 datasets and near-SOTA accuracy on another 4 (see Table 3 ). These results highlight the effectiveness of embedding fusion from diverse foundation models in improving performance across a broad range of biomedical image classification tasks. View this table: View inline View popup Download powerpoint Table 3. Performance Comparison Between BioFuse and Best Existing Methods on MedMNIST+. Bold values indicate the best performance for each dataset. A particularly interesting finding is the versatility of histopathology and radiology foundation models. Histopathology models UNI, Hibou-B, CONCH and Prov-GigaPath and radiology models CheXagent and rad-dino consistently appear in the top-performing model combinations, even when applied to modalities different from their pre-training data. This crossmodality effectiveness suggests these models learn generalisable features useful across various biomedical imaging tasks. Cross-Modal Transfer Performance Transfer to other medical imaging modalities We observed notable cross-modal transfer capabilities across both histopathology and radiology models when evaluating each model independently across the datasets, as illustrated in Fig 4 and Fig 5 . This single-model evaluation used fixed hyperparameters for XGBoost as described in Experimental Procedure. As a baseline, we included CLIP [ 32 ], a general-purpose vision-language foundation model, to compare performance against biomedical foundation models. CLIP’s performance was generally low across all datasets, with the notable exception of OrganAMNIST, where it ranked in the top 3 for both accuracy and AUC. Download figure Open in new tab Fig 4. Heatmap of test AUC, showing single model performance across MedMNIST+ datasets. CLIP is included as a baseline general-purpose foundation model for comparison. White box indicates the top-performer while red boxes highlight the 2nd and 3rd top performers for each dataset; More than three models may be highlighted if they share identical values. Download figure Open in new tab Fig 5. Heatmap showing test accuracy, showing single model performance across MedMNIST+ datasets and ImageNet-1K (top-1). CLIP is included as a baseline general-purpose foundation model for comparison. White boxes indicate top performers, while red boxes highlight second and third best performers for each dataset. When multiple models achieve identical performance, more than three boxes may be highlighted. Fig 4 shows that Prov-GigaPath, Hibou-B, and UNI demonstrated strong performance in various radiology datasets. Prov-GigaPath achieved the highest AUC on BreastMNIST and ranked among the top-3 for OrganAMNIST. Hibou-B secured top-3 AUC in BreastMNIST, OrganCMNIST and OrganSMNIST. UNI excelled in CT and X-ray tasks, achieving the highest AUC for OrganSMNIST and top-3 in PneumoniaMNIST. CheXagent, primarily trained on chest X-rays, demonstrated impressive cross-modal capabilities, achieving top performance in DermaMNIST and ranking among the top-3 for BloodMNIST, TissueMNIST, and RetinaMNIST. Fig 5 corroborates this trend, with Hibou-B achieving top-3 accuracy on both OrganCMNIST and OrganSMNIST. Prov-GigaPath demonstrated excellence in radiology tasks, achieving top accuracy for BreastMNIST and ranking in top-3 for OrganAMNIST, OrganCMNIST, OrganSMNIST, and PneumoniaMNIST. UNI showed remarkable performance, tying for top accuracy in BreastMNIST and leading in OrganSMNIST. CheXagent’s versatility is again evident, achieving top accuracy in both DermaMNIST and RetinaMNIST, while securing top-3 positions in BloodMNIST and TissueMNIST. Transfer to Natural Images To comprehensively evaluate the generalisation capabilities of medical foundation models, we assess their performance on the ImageNet-1K [ 77 ] test set, a benchmark dataset widely used in computer vision. While we don’t expect medical models to match the performance of models trained specifically on natural images, their performance provides valuable insights into their general visual understanding capabilities. Surprisingly, some medical models demonstrate remarkable transfer ability, with Prov-GigaPath and CheXAgent achieving 78.2% and 66.0% Top-1 accuracy respectively, outperforming CLIP (58.1%) despite CLIP being trained on 400M natural image-text pairs. Table 4 presents the complete ImageNet-1K test set performance results. View this table: View inline View popup Download powerpoint Table 4. Performance Comparison of Biomedical Foundation Models on ImageNet-1K. Bold values indicate best performance. Discussion Key Findings Based on the comprehensive results in Table 3 , we would like to highlight some important observations: State-of-the-Art Performance: BioFuse demonstrates exceptional performance on the MedMNIST+ benchmark, achieving new stateof-the-art (SOTA) scores across multiple datasets. Specifically, it achieves the highest AUC in five datasets and the best accuracy in two datasets. These improvements, achieved in a highly competitive benchmark, underscore BioFuse’s ability to extract and combine more informative features from multiple pre-trained models. Strong Performance on DermaMNIST: A notable improvement was observed on the DermaMNIST dataset, where BioFuse achieved state-of-the-art performance in both AUC and accuracy. Despite the SOTA AUC already being at 0.97, BioFuse managed to deliver a further performance improvement in AUC and 1.4% in Accuracy. This demonstrates BioFuse’s ability to improve performance even on tasks where the margin for improvement is small. Near SOTA AUC performance: BioFuse demonstrated highly competitive performance across multiple datasets, achieving AUC scores within 1.5% of state-of-the-art methods. This consistent performance was observed across various CT imaging tasks (OrganAMNIST, OrganCMNIST, OrganSMNIST) and microscopy datasets (TissueMNIST, PathMNIST). While BioFuse did not achieve the highest test accuracy for most datasets, it notably achieved SOTA performance on PathMNIST. The minimal gap in AUC scores compared to bestperforming methods suggests BioFuse maintains robust class separation capabilities across these diverse medical imaging tasks. AUC Superiority in Lower Accuracy Datasets: For datasets such as OCTMNIST, PneumoniaMNIST, and ChestMNIST, BioFuse’s test accuracy lagged behind existing models, but it outperformed them in AUC scores. Specifically, BioFuse achieved an AUC of 0.994 (vs. 0.992) for OCTMNIST, 0.997 (vs. 0.995) for PneumoniaMNIST, and 0.839 (vs. 0.822) for ChestMNIST, showing that even when accuracy lags, BioFuse’s embeddings allow the model to make confident and accurate class distinctions. Efficacy of Model Ensembles: Across all datasets, the bestperforming combinations involved multiple foundation models. No single model was able to outperform the ensembles, underscoring the importance of model diversity in feature representation. This finding suggests that the combined outputs from different models allow BioFuse to better capture complementary information across multiple modalities, resulting in more robust performance across a variety of biomedical imaging tasks. Cross-Modal Transfer Biomedical foundation models exhibited remarkable cross-modal transfer capabilities, often excelling in tasks well beyond their original training domains. This generalisation likely stems from shared visual feature representations that transcend specific imaging modalities. Notably, models trained on a single modality (such as histopathology-specific models) demonstrated stronger cross-modal transfer than those trained on multiple modalities, suggesting a potential benefit to focused, modality-specific pre-training for developing generalisable representations. Histopathology models — particularly UNI, Hibou-B, and Prov-GigaPath — demonstrated exceptional cross-modal performance, likely due to their Vision Transformer-based architectures. These models develop capacity to process multi-scale visual features during pre-training, from micron-scale cellular details to millimeter-scale tissue structures. This multi-scale capability appears particularly transferable across modalities, as the feature extraction mechanisms that identify cellular boundaries and tissue organization in histopathology may transfer effectively to detecting analogous structural patterns in retinal images and radiological scans, despite differences in visual appearance. CheXagent’s sophisticated architecture, which integrates an 8-billion parameter clinical language model with a vision encoder and vision-language bridge, may contribute to its strong cross-modal adaptability. Pre-trained on 6 million instruction-image-answer triplets across 65 diverse datasets, this model appears to develop both modality-specific and modality-agnostic feature representations. The language component potentially functions as a semantic intermediary, facilitating knowledge transfer between distinct imaging domains. This architectural advantage correlates with CheXagent’s impressive performance in non-radiological tasks such as dermatology (AUC 0.960) and microscopy (AUC 0.999). While our findings demonstrate substantial cross-modal capabilities, this transfer is not universal across all biomedical tasks. We observed performance decreases in highly specialized domains such as CT scan interpretation, where OrganAMNIST and OrganCMNIST showed the widest performance gaps between specialized and cross-modal models. These limitations suggest that effective cross-modal transfer depends on both model architecture and the inherent similarity between source and target domains. Future work should explore these boundaries systematically, potentially guiding the development of more universally transferable biomedical foundation models. Computational Considerations Understanding BioFuse’s computational demands is essential for practical implementation. Following [ 78 ], we analyzed total response time as a realistic computational performance metric. Evaluating BioFuse across 12 datasets required 47h 19m for embedding extraction and 15h 35m for model evaluation, totaling 62h 54m . Fig 6 presents a detailed breakdown of computation time across all MedMNIST+ datasets. The distribution reveals significant performance variation based on dataset characteristics. TissueMNIST demanded the most processing time (17.8 hours), primarily due to its large size. In contrast, smaller datasets like RetinaMNIST and BreastMNIST required less than 0.5 hours each for complete processing. Download figure Open in new tab Fig 6. Stacked bar chart showing the total evaluation time for each dataset. The chart is divided into embedding extraction and XGBoost evaluation phases. The blue section represents the time spent on embedding extraction, while the red section indicates the time for model training and validation using XGBoost. The data reveals that embedding extraction represents the dominant computational bottleneck, accounting for approximately 75% of total processing time. This bottleneck scales primarily with the size of the dataset, as clearly demonstrated by TissueMNIST (236,386 samples) which require the longest extraction time, followed by OCTMNIST (109,309 samples) and PathMNIST (107,180 samples). The relationship between computation time and dataset size is largely linear for each model, as all images are standardized to 224×224 resolution. In particular, ChestMNIST exhibits an unusually high proportion of time devoted to XGBoost evaluation relative to other datasets (approximately 40% of total processing time versus typical 20-25%). This anomaly stems primarily from ChestMNIST’s multi-label classification nature with 14 possible labels, substantially increasing the complexity of the classification task compared to binary or single-label multi-class problems. Despite using high-dimensional concatenated embeddings, XGBoost training and evaluation remained computationally efficient in most cases, typically requiring less than 25% of the total computation time. This reinforces XGBoost’s reputation for handling high-dimensional data effectively. Our analysis of GPU memory requirements showed that 7 of 9 foundation models operated within 5GB of VRAM, making them suitable for consumer-grade GPUs (Appendix C Fig 7 ). However, CheXagent, with 8 billion parameters, required 33.8GB VRAM, necessitating server-grade GPU hardware (Appendix C Fig 8 ). Although memory requirements scale with data set size, the relative differences between models remain consistent. These resource considerations are particularly important for researchers planning to implement BioFuse in environments with limited computational infrastructure. Download figure Open in new tab Fig 7. GPU VRAM usage (in GB) for each model when processing the RetinaMNIST dataset. CheXagent has the highest memory demand, requiring more than 33 GB of VRAM for the 1,080 training samples. Download figure Open in new tab Fig 8. Embedding extraction time (in mins) for each model when processing the RetinaMNIST dataset. CheXagent takes the longest extraction time, significantly exceeding that of other models for the 1,080 training samples. To support reproducibility and eliminate the need for time-intensive embedding extraction, we provide the complete embedding cache via Zenodo for MedMNIST+ [ 79 ] and ImageNet-1K [ 80 ]. Limitations and Future Directions While BioFuse demonstrates strong potential, several limitations must be addressed in future research: Evaluation Scope and Task Generalisation: BioFuse is currently evaluated only on classification tasks (binary, multi-class, and multi-label) using MedMNIST+. Testing on broader task types (e.g., segmentation, object detection, regression) and real-world clinical datasets would provide more comprehensive validation. Future work should explore BioFuse’s applicability to time-series medical data and multimodal clinical records, potentially expanding its utility in diagnostic and prognostic applications. Dependency on Pre-trained Models: BioFuse’s performance inherently depends on the quality and diversity of its foundation models. If these models lack generalisability or introduce biases, these issues may propagate through the fusion process. Expanding the model pool to include diverse biomedical domains (e.g., genomics, MRI, ultrasound, and clinical text) could mitigate this limitation and enable truly multimodal integration beyond imaging. Computational Overhead: Extracting embeddings from multiple models and performing exhaustive grid searches requires significant GPU resources, particularly when using advanced models like CheXagent (33.8GB VRAM). Developing more efficient search strategies that can identify optimal model combinations without exhaustively evaluating all possibilities represents another promising direction. Vector Concatenation Limitations: The current approach can lead to high-dimensional embeddings with redundancy, potentially increasing overfitting risks on smaller datasets. Exploring alternative fusion strategies, such as attention-based fusion, knowledge distillation, or learned projection spaces, may improve efficiency and reduce dimensionality issues while preserving information content. Interpretability Challenges: Combining multiple models reduces transparency in how predictions are formed, which is critical for clinical applications. Integrating explainability techniques like saliency maps, feature attribution, or attention visualization could improve interpretability, helping clinicians understand feature contributions from different models. This remains essential for responsible deployment in healthcare settings. By addressing these areas through interdisciplinary collaboration between AI researchers, clinicians, and domain experts, BioFuse can evolve into a more efficient, interpretable, and generalisable tool for multimodal biomedical research and clinical applications. Conclusion The integration of diverse foundation models represents a promising frontier for advancing biomedical imaging analysis. In this work, we introduced BioFuse, a novel framework that systematically fuses embeddings from multiple biomedical foundation models to generate optimised representations for downstream tasks. Evaluated across 12 diverse imaging modalities in the MedMNIST+ benchmark, BioFuse with XGBoost classification outperformed existing methods, achieving the highest AUC in five datasets and maintaining near-SOTA performance in most others. Notably, it demonstrated exceptional performance in dermatology classification (DermaMNIST) and revealed unexpected cross-modal transfer capabilities in histopathology and radiology models like UNI, Hibou-B, Prov-GigaPath, and CheXagent. These results highlight the benefits of leveraging multiple pre-trained models rather than relying on a single foundation model. BioFuse’s ability to automatically identify and integrate complementary representations from diverse models suggests significant potential for healthcare applications requiring comprehensive image interpretation across modalities. The framework’s extensible architecture ensures adaptability to future foundation models as they emerge. While demonstrating clear advantages, BioFuse faces challenges, including computational overhead from grid search and potential redundancy in concatenated embeddings. Future work should explore more efficient fusion strategies, expand applications beyond classification to segmentation and detection tasks, and incorporate interpretability mechanisms essential for clinical adoption and regulatory approval. By harnessing the collective strengths of multiple foundation models through a systematic approach to embedding fusion, BioFuse not only improves performance on benchmark tasks but also opens new avenues for cross-modal knowledge transfer in biomedical imaging. This contribution moves us closer to more reliable and comprehensive AI-assisted medical decision-making systems that can integrate information across the diverse imaging modalities encountered in clinical practice. Funding This research was supported by the School of Computer Science at the University of St Andrews through a PhD studentship. The funder had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript. Competing Interests The authors have declared that no competing interests exist. Ethics Statement This research was conducted under ethical approval from the institutional review board (approval code CS17485). Supporting information S1 Appendix. BioFuse API Specification Typical workflow BioFuse’s workflow is split into 5 stages: 1. Initialize BioFuse with selected foundation models. 2. Generate BioFuse embeddings for training and validation datasets. 3. Train and tune downstream model using the BioFuse embeddings. 4. Generate test embeddings using the BioFuseModel embedding generator 4. Evaluate the performance of the final model using test embeddings. High-level API 1. Initialize BioFuse Download figure Open in new tab 2. Generate BioFuse Embeddings Generate embeddings from the training and validation sets (X,y) pairs along with a BioFuseModel that can be used for generating embeddings for new data. BioFuse also expects the task type, currently supporting binary, multi-class and multi-label classification. Download figure Open in new tab 3. Generate BioFuse Embeddings for new data using BioFuseModel Download figure Open in new tab Example Usage Download figure Open in new tab S2 Appendix. Hyperparameter Tuning Configurations for XGBoost Overview This appendix provides the specific hyperparameter configurations used for XGBoost on each dataset in MedMNIST+. Given XGBoost’s extensive hyperparameter tuning space, we employed the Bayesian optimization method through the Weights Biases (wandb) platform. This approach allowed us to efficiently navigate the vast hyperparameter landscape and identify optimal configurations for each dataset, balancing the exploration of the parameter space with the exploitation of promising regions. Hyperparameter Tuning setup Table 5 presents the hyperparameters considered and their respective search ranges. For each parameter, we selected ranges that encompass both conservative and aggressive values to thoroughly explore the model’s behavior. View this table: View inline View popup Download powerpoint Table 5. XGBoost Hyperparameters and Their Search Ranges View this table: View inline View popup Download powerpoint Table 6. Optimal XGBoost Hyperparameter Configurations for MedMNIST+ 2D Dataset specific configurations After conducting an extensive hyperparameter search, we identified the optimal configurations for each dataset in the MedMNIST collection. Table 6 presents these optimised hyperparameter settings, which show interesting variations across different medical imaging tasks. S3 Appendix. Computation Time S4 Appendix. Pretraining Datasets for Biomedical Foundation Models The table below details the datasets used in the pretraining of biomedical foundation models used by BioFuse. This ensures transparency regarding the sources and content of each dataset, clarifying any overlaps or distinctions between the training data of individual models. Such information is critical for evaluating dataset independence, particularly in relation to BioFuse’s target datasets, like MedMNIST. View this table: View inline View popup Download powerpoint Table 7. Pretraining Datasets for Models in BioFuse S5 Appendix. MedMNIST+ Dataset Details The MedMNIST+ collection provides a diverse set of labeled medical imaging datasets across various diagnostic domains, designed to support machine learning and deep learning research in healthcare. The table below outlines the source datasets, imaging modalities, and primary diagnostic categories for each MedMNIST+ subset, highlighting the origins and clinical relevance of each dataset. Importantly, this table serves as evidence that none of the datasets in MedMNIST+ overlap with the pretraining datasets used in the biomedical foundation models integrated within BioFuse. View this table: View inline View popup Download powerpoint Table 8. MedMNIST Datasets and Their Sources Acknowledgments We thank the School of Computer Science at the University of St Andrews for providing computational resources and support. We also appreciate the IT team for their technical assistance. Footnotes ↵ ☯ Led research: conceptualization, methodology, software, formal analysis, and drafting. ↵ ‡ Supervised research and provided critical revisions. ‡ dcchb{at}st-andrews.ac.uk ↵ 1 Code: https://github.com/mnhcorp/biofuse References 1. ↵ He J , Baxter SL , Xu J , Xu J , Zhou X , Zhang K . The practical implementation of artificial intelligence technologies in medicine . Nature Medicine . 2019 ; 25 ( 1 ): 30 – 36 . OpenUrl CrossRef PubMed 2. Topol EJ . High-performance medicine: the convergence of human and artificial intelligence . Nature Medicine . 2019 ; 25 ( 1 ): 44 – 56 . OpenUrl CrossRef PubMed 3. ↵ Chen X , Wang X , Zhang K , Fung KM , Thai TC , Moore K , et al. Recent advances and clinical applications of deep learning in medical image analysis . Medical Image Analysis . 2022 ; 79 : 102444 . doi: 10.1016/j.media.2022.102444 . OpenUrl CrossRef PubMed 4. ↵ Bommasani R , Hudson DA , Adeli E , Altman R , Arora S , von Arx S , et al. On the opportunities and risks of foundation models . arXiv preprint arXiv:210807258. 2021 ;. 5. ↵ Azad B , Azad R , Eskandari S , Bozorgpour A , Kazerouni A , Rekik I , et al. Foundational Models in Medical Imaging: A Comprehensive Survey and Future Vision ; 2023 . Available from: https://arxiv.org/abs/2310.18689. 6. ↵ Wang Z , Wu Z , Agarwal D , Sun J. MedCLIP: Contrastive Learning from Unpaired Medical Images and Text ; 2022 . Available from: https://arxiv.org/abs/2210.10163 . 7. ↵ Ma J , He Y , Li F , Han L , You C , Wang B . Segment anything in medical images . Nature Communications . 2024 ; 15 ( 1 ): 654 . OpenUrl CrossRef PubMed 8. ↵ Breen J , Allen K , Zucker K , Godson L , Orsi NM , Ravikumar N. Histopathology Foundation Models Enable Accurate Ovarian Cancer Subtype Classification ; 2024 . Available from: https://arxiv.org/abs/2405.09990 . 9. ↵ Vorontsov E , Bozkurt A , Casson A , Shaikovski G , Zelechowski M , Liu S , et al. Virchow: A Million-Slide Digital Pathology Foundation Model ; 2023 . Available from: https://arxiv.org/abs/2309.07778 . 10. ↵ Saab K , Tu T , Weng WH , Tanno R , Stutz D , Wulczyn E , et al. Capabilities of Gemini Models in Medicine ; 2024 . Available from: https://arxiv.org/abs/2404.18416 . 11. ↵ Lee J , Yoon W , Kim S , Kim D , Kim S , So CH , et al. BioBERT: a pre-trained biomedical language representation model for biomedical text mining . Bioinformatics . 2020 ; 36 ( 4 ): 1234 – 1240 . OpenUrl CrossRef PubMed 12. ↵ Luo R , Sun L , Xia Y , Qin T , Zhang S , Poon H , et al. BioGPT: generative pre-trained transformer for biomedical text generation and mining . Briefings in bioinformatics . 2022 ; 23 ( 6 ):bbac409. 13. ↵ Ji Y , Zhou Z , Liu H , Davuluri RV . DNABERT: pre-trained Bidirectional Encoder Representations from Transformers model for DNAlanguage in genome . Bioinformatics . 2021 ; 37 ( 15 ): 2112 – 2120 . OpenUrl CrossRef PubMed 14. ↵ Yang F , Wang W , Wang F , Fang Y , Tang D , Huang J , et al. scBERT as a large-scale pretrained deep language model for cell type annotation of single-cell RNA-seq data . Nature Machine Intelligence . 2022 ; 4 ( 10 ): 852 – 866 . OpenUrl CrossRef 15. ↵ Wang Z , Liu C , Zhang S , Dou Q. In: Foundation Model for Endoscopy Video Analysis via Large-Scale Self-supervised Pre-train . Springer Nature Switzerland ; 2023 . p. 101–111. Available from : doi: 10.1007/978-3-031-43996-4_10 . OpenUrl CrossRef 16. ↵ Wölflein G , Ferber D , Meneghetti AR , Nahhas OSME , Truhn D , Carrero ZI , et al. Benchmarking Pathology Feature Extractors for Whole Slide Image Classification ; 2024 . Available from: https://arxiv.org/abs/2311.11772 . 17. ↵ Tiu E , Talius E , Patel P , Langlotz CP , Ng AY , Rajpurkar P . Expertlevel detection of pathologies from unannotated chest X-ray images via self-supervised learning . Nature Biomedical Engineering . 2022 ; 6 ( 12 ): 1399 – 1406 . OpenUrl CrossRef PubMed 18. Wang X , Yang S , Zhang J , Wang M , Zhang J , Yang W , et al. Transformer-based unsupervised contrastive learning for histopathological image classification . Medical Image Analysis . 2022 ; 81 : 102559 . doi: 10.1016/j.media.2022.102559 . OpenUrl CrossRef PubMed 19. ↵ Singhal K , Tu T , Gottweis J , Sayres R , Wulczyn E , Hou L , et al. Towards Expert-Level Medical Question Answering with Large Language Models ; 2023 . Available from: https://arxiv.org/abs/2305.09617 . 20. ↵ Khan W , Leem S , See KB , Wong JK , Zhang S , Fang R. A Comprehensive Survey of Foundation Models in Medicine ; 2024 . Available from: https://arxiv.org/abs/2406.10729 . 21. ↵ Zhang Y , Jiang H , Miura Y , Manning CD , Langlotz CP. Contrastive Learning of Medical Visual Representations from Paired Images and Text ; 2022 . Available from: https://arxiv.org/abs/2010.00747 . 22. ↵ Li W , Peng Y , Zhang M , Ding L , Hu H , Shen L. Deep Model Fusion: A Survey ; 2023 . Available from: https://arxiv.org/abs/2309.15698 . 23. ↵ Chen MF , Fu DY , Adila D , Zhang M , Sala F , Fatahalian K , et al. Shoring Up the Foundations: Fusing Model Embeddings and Weak Supervision ; 2022 . Available from: https://arxiv.org/abs/2203.13270 . 24. ↵ Neidlinger P , Nahhas OSME , Muti HS , Lenz T , Hoffmeister M , Brenner H , et al. Benchmarking foundation models as feature extractors for weakly-supervised computational pathology ; 2024 . Available from: https://arxiv.org/abs/2408.15823 . 25. ↵ Qiu J , Li L , Sun J , Peng J , Shi P , Zhang R , et al. Large AI Models in Health Informatics: Applications, Challenges, and the Future . IEEE Journal of Biomedical and Health Informatics . 2023 ; 27 ( 12 ): 6074 – 6087 . doi: 10.1109/jbhi.2023.3316750 . OpenUrl CrossRef 26. ↵ Brown TB , Mann B , Ryder N , Subbiah M , Kaplan J , Dhariwal P , et al. Language Models are Few-Shot Learners ; 2020 . Available from: https://arxiv.org/abs/2005.14165 . 27. ↵ Vaswani A , Shazeer N , Parmar N , Uszkoreit J , Jones L , Gomez AN , et al. Attention Is All You Need ; 2023 . Available from: https://arxiv.org/abs/1706.03762 . 28. ↵ Kaplan J , McCandlish S , Henighan T , Brown TB , Chess B , Child R , et al. Scaling Laws for Neural Language Models ; 2020 . Available from: https://arxiv.org/abs/2001.08361 . 29. ↵ Weinstein JN , Collisson EA , Mills GB , Shaw KRM , Ozenberger BA , Ellrott K , et al. The Cancer Genome Atlas PanCancer analysis project . Nature Genetics . 2013 ; 45 ( 10 ): 1113 – 1120 . doi: 10.1038/ng.2764 . OpenUrl CrossRef PubMed 30. ↵ Xie Y , Zhou C , Gao L , Wu J , Li X , Zhou HY , et al. MedTrinity-25M: A Large-scale Multimodal Dataset with Multigranular Annotations for Medicine ; 2024 . Available from: https://arxiv.org/abs/2408 . 02900. 31. ↵ Shoeybi M , Patwary M , Puri R , LeGresley P , Casper J , Catanzaro B. Megatron-LM : Training Multi-Billion Parameter Language Models Using Model Parallelism ; 2020 . Available from: https://arxiv.org/abs/1909.08053 . 32. ↵ Radford A , Kim JW , Hallacy C , Ramesh A , Goh G , Agarwal S , et al. Learning Transferable Visual Models From Natural Language Supervision ; 2021 . Available from: https://arxiv.org/abs/2103 . 00020. 33. ↵ Devlin J , Chang MW , Lee K , Toutanova K. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding ; 2019 . Available from: https://arxiv.org/abs/1810.04805 . 34. ↵ Myles C , Um IH , Harrison DJ , Harris-Birtill D . Leveraging Foundation Models for Enhanced Detection of Colorectal Cancer Biomarkers in Small Datasets . In: Yap MH , Kendrick C , Behera A , Cootes T , Zwiggelaar R , editors. Medical Image Understanding and Analysis . Cham : Springer Nature Switzerland ; 2024 . p. 329 – 343 . 35. ↵ Chen T , Kornblith S , Norouzi M , Hinton G. A Simple Framework for Contrastive Learning of Visual Representations ; 2020 . Available from: https://arxiv.org/abs/2002.05709 . 36. ↵ He K , Chen X , Xie S , Li Y , Dollár P , Girshick R. Masked Autoencoders Are Scalable Vision Learners ; 2021 . Available from: https://arxiv.org/abs/2111.06377 . 37. ↵ Oquab M , Darcet T , Moutakanni T , Vo H , Szafraniec M , Khalidov V , et al. DINOv2: Learning Robust Visual Features without Supervision ; 2024 . Available from: https://arxiv.org/abs/2304.07193 . 38. ↵ Caron M , Touvron H , Misra I , Jégou H , Mairal J , Bojanowski P , et al. Emerging Properties in Self-Supervised Vision Transformers ; 2021 . Available from: https://arxiv.org/abs/2104.14294 . 39. ↵ Chen RJ , Ding T , Lu MY , Williamson DFK , Jaume G , Song AH , et al. Towards a general-purpose foundation model for computational pathology . Nature Medicine . 2024 ; 30 ( 3 ): 850 – 862 . doi: 10.1038/s41591-024-02857-3 . OpenUrl CrossRef 40. ↵ Lab M. UNI2: Pathology Foundation Model ; 2025 . https://github.com/mahmoodlab/UNI/tree/uni2 . 41. ↵ Xu H , Usuyama N , Bagga J , Zhang S , Rao R , Naumann T , et al. A whole-slide foundation model for digital pathology from real-world data . Nature . 2024 ; 630 (8015):181–188. doi: 10.1038/s41586-024-07441-w . OpenUrl CrossRef PubMed 42. ↵ Nechaev D , Pchelnikov A , Ivanova E. Hibou: A Family of Foundational Vision Transformers for Pathology ; 2024 . Available from: https://arxiv.org/abs/2406.05074 . 43. ↵ Pérez-García F , Sharma H , Bond-Taylor S , Bouzid K , Salvatelli V , Ilse M , et al. RAD-DINO: Exploring Scalable Medical Image Encoders Beyond Text Supervision ; 2024 . Available from: https://arxiv.org/abs/2401.10815 . 44. ↵ Zhang S , Xu Y , Usuyama N , Xu H , Bagga J , Tinn R , et al. BiomedCLIP: a multimodal biomedical foundation model pretrained from fifteen million scientific image-text pairs ; 2024 . Available from: https://arxiv.org/abs/2303.00915 . 45. ↵ Eslami S , Meinel C , de Melo G. PubMedCLIP: How Much Does CLIP Benefit Visual Question Answering in the Medical Domain ? In: Vlachos A , Augenstein I , editors. Findings of the Association for Computational Linguistics: EACL 2023 . Dubrovnik, Croatia : Association for Computational Linguistics ; 2023. p. 1181–1193. Available from: https://aclanthology.org/2023.findings-eacl.88 . 46. ↵ Chen Z , Varma M , Delbrouck JB , Paschali M , Blankemeier L , Veen DV , et al. CheXagent: Towards a Foundation Model for Chest XRay Interpretation ; 2024 . Available from: https://arxiv.org/abs/2401.12208 . 47. ↵ Lu MY , Chen B , Williamson DFK , Chen RJ , Liang I , Ding T , et al. Towards a Visual-Language Foundation Model for Computational Pathology ; 2023 . Available from: https://arxiv.org/abs/2307.12914 . 48. ↵ Zarif S , Abdulkader H , Elaraby I , Alharbi A , Elkilani WS , Pławiak P . Using hybrid pre-trained models for breast cancer detection . PLOS ONE . 2024 ; 19 ( 1 ): e0296912 . doi: 10.1371/journal.pone.0296912 . OpenUrl CrossRef PubMed 49. ↵ Dong X , Li M , Zhou P , Deng X , Li S , Zhao X , et al. Fusing pretrained convolutional neural networks features for multi-differentiated subtypes of liver cancer on histopathological images . BMC Medical Informatics and Decision Making . 2022 ; 22 ( 1 ). doi: 10.1186/s12911-022-01798-6 . OpenUrl CrossRef 50. ↵ Sharma S , au2 RDJ. BioFLAIR: Pretrained Pooled Contextualized Embeddings for Biomedical Sequence Labeling Tasks ; 2019 . Available from: https://arxiv.org/abs/1908.05760 . 51. ↵ Yogarajan V , Pfahringer B , Smith T , Montiel J. Improving Predictions of Tail-end Labels using Concatenated BioMed-Transformers for Long Medical Documents ; 2021 . Available from: https://arxiv.org/abs/2112.01718. 52. ↵ Zayats V , Chen P , Ferrari M , Padfield D. Zipper: A Multi-Tower Decoder Architecture for Fusing Modalities ; 2024 . Available from: https://arxiv.org/abs/2405.18669 . 53. ↵ Wang Z , Wang Z , Srinivasan B , Ioannidis VN , Rangwala H , Anubhai R. BioBridge: Bridging Biomedical Foundation Models via Knowledge Graphs ; 2024 . Available from: https://arxiv.org/abs/2310.03320 . 54. ↵ Wang X , Jiang Y , Bach N , Wang T , Huang Z , Huang F , et al. Automated Concatenation of Embeddings for Structured Prediction ; 2021 . Available from: https://arxiv.org/abs/2010.05006 . 55. ↵ Alain G , Bengio Y. Understanding intermediate layers using linear classifier probes ; 2018 . Available from: https://arxiv.org/abs/1610.01644 . 56. ↵ Chen T , Guestrin C. XGBoost: A Scalable Tree Boosting System . In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining . KDD’16. ACM; 2016 .Available from : doi: 10.1145/2939672.2939785 . OpenUrl CrossRef 57. ↵ Jiang W , Synovic N , Hyatt M , Schorlemmer TR , Sethi R , Lu YH , et al. An Empirical Study of Pre-Trained Model Reuse in the Hugging Face Deep Learning Model Registry . In: 2023 IEEE/ACM 45th International Conference on Software Engineering (ICSE) ; 2023 . p. 2463–2475. doi: 10.1109/ICSE48619.2023.00206 . OpenUrl CrossRef 58. Ouyang L , Wu J , Jiang X , Almeida D , Wainwright CL , Mishkin P , et al. Training language models to follow instructions with human feedback ; 2022 . Available from: https://arxiv.org/abs/2203.02155 . 59. ↵ Wolf T , Debut L , Sanh V , Chaumond J , Delangue C , Moi A , et al. HuggingFace’s Transformers: State-of-the-art Natural Language Processing ; 2020 . Available from: https://arxiv.org/abs/1910.03771 . 60. ↵ Ilharco G , Wortsman M , Wightman R , Gordon C , Carlini N , Taori R , et al. OpenCLIP ; 2021 . Available from : doi: 10.5281/zenodo.5143773 . OpenUrl CrossRef 61. ↵ Wightman R. PyTorch Image Models ; 2019 . https://github.com/rwightman/pytorch-image-models . 62. ↵ Yang J , Shi R , Wei D , Liu Z , Zhao L , Ke B , et al. MedMNIST v2 A large-scale lightweight benchmark for 2D and 3D biomedical image classification . Scientific Data . 2023 ; 10 ( 1 ). doi: 10.1038/s41597-022-01721-8 . OpenUrl CrossRef PubMed 63. Kather JN , Krisam J , Charoentong P , Luedde T , Herpel E , Weis CA , et al. Predicting survival from colorectal cancer histology slides using deep learning: A retrospective multicenter study . PLoS medicine . 2019 ; 16 ( 1 ): e1002730 . OpenUrl CrossRef 64. Wang X , Peng Y , Lu L , Lu Z , Bagheri M , Summers RM . Chestxray8: Hospital-scale chest x-ray database and benchmarks on weaklysupervised classification and localization of common thorax diseases . In: Proceedings of the IEEE conference on computer vision and pattern recognition ; 2017 . p. 2097 – 2106 . 65. Tschandl P , Rosendahl C , Kittler H . The HAM10000 dataset, a large collection of multi-source dermatoscopic images of common pigmented skin lesions . Scientific data . 2018 ; 5 ( 1 ): 1 – 9 . OpenUrl CrossRef PubMed 66. Kermany DS , Goldbaum M , Cai W , Valentim CC , Liang H , Baxter SL , et al. Identifying medical diagnoses and treatable diseases by image-based deep learning . cell . 2018 ; 172 ( 5 ): 1122 – 1131 . OpenUrl CrossRef PubMed 67. Liu R , Wang X , Wu Q , Dai L , Fang X , Yan T , et al. DeepDRiD: Diabetic Retinopathy—Grading and Image Quality Estimation Challenge . Patterns . 2022 ; 3 ( 6 ): 100512 . doi: 10.1016/j.patter.2022.100512 . OpenUrl CrossRef PubMed 68. Al-Dhabyani W , Gomaa M , Khaled H , Fahmy A . Dataset of breast ultrasound images . Data in brief . 2020 ; 28 : 104863 . 69. Acevedo A , Merino González A , Alférez Baquero ES , Molina Borrás Á , Boldú Nebot L , Rodellar Benedé J . A dataset of microscopic peripheral blood cell images for development of automatic recognition systems . Data in brief . 2020 ; 30 (article 105474). 70. Ljosa V , Sokolnicki KL , Carpenter AE . Annotated high-throughput microscopy image sets for validation . Nature methods . 2012 ; 9 ( 7 ): 637 – 637 . OpenUrl CrossRef PubMed 71. Bilic P , Christ P , Li HB , Vorontsov E , Ben-Cohen A , Kaissis G , et al. The liver tumor segmentation benchmark (lits) . Medical Image Analysis . 2023 ; 84 : 102680 . 72. ↵ Gutiérrez PA , Perez-Ortiz M , Sanchez-Monedero J , FernandezNavarro F , Hervas-Martinez C . Ordinal regression methods: survey and experimental study . IEEE Transactions on Knowledge and Data Engineering . 2016 ; 28 ( 1 ): 127 – 146 . OpenUrl CrossRef 73. ↵ XGBoost Parameters #x 2014 ; xgboost 2.1.1 documentation — xgboost.readthedocs.io;. https://xgboost.readthedocs.io/en/stable/parameter.html . 74. ↵ Classification : ROC and AUC | Machine Learning | Google for Developers — developers.google.com; https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc . 75. Doerrich S , Salvo FD , Brockmann J , Ledig C. Rethinking Model Prototyping through the MedMNIST+ Dataset Collection ; 2024 . Available from: https://arxiv.org/abs/2404.15786 . 76. Manzari ON , Ahmadabadi H , Kashiani H , Shokouhi SB , Ayatollahi A . MedViT: A robust vision transformer for generalized medical image classification . Computers in Biology and Medicine . 2023 ; 157 : 106791 . doi: 10.1016/j.compbiomed.2023.106791 . OpenUrl CrossRef 77. ↵ Russakovsky O , Deng J , Su H , Krause J , Satheesh S , Ma S , et al. ImageNet Large Scale Visual Recognition Challenge ; 2015 . Available from: https://arxiv.org/abs/1409.0575 . 78. ↵ Harris-Birtill D , Harris-Birtill R . Understanding computation time: a critical discussion of time as a computational performance metric . In: Misztal A , Harris PA , Parker JA , editors. Time in variance. The study of time, vol. 17 . Leiden : Brill ; 2021 . p. 220–248. doi: 10.1163/9789004470170_014 . OpenUrl CrossRef 79. ↵ Hossain MN , Harris-Birtill D . BioFuse Embedding Cache for MedMNIST+ 2D Datasets ; 2025 . Available from : doi: 10.5281/zenodo.13952293 . OpenUrl CrossRef 80. ↵ Hossain MN , Harris-Birtill D . BioFuse Embedding Cache for ImageNet-1K; 2025 . Available from : doi: 10.5281/zenodo.14930584 . View the discussion thread. Back to top Previous Next Posted March 07, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following BioFuse: An embedding fusion framework for biomedical foundation models Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share BioFuse: An embedding fusion framework for biomedical foundation models Mirza Nasir Hossain , David Harris-Birtill bioRxiv 2025.03.01.640976; doi: https://doi.org/10.1101/2025.03.01.640976 Share This Article: Copy Citation Tools BioFuse: An embedding fusion framework for biomedical foundation models Mirza Nasir Hossain , David Harris-Birtill bioRxiv 2025.03.01.640976; doi: https://doi.org/10.1101/2025.03.01.640976 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7637) Biochemistry (17705) Bioengineering (13899) Bioinformatics (41968) Biophysics (21460) Cancer Biology (18603) Cell Biology (25526) Clinical Trials (138) Developmental Biology (13385) Ecology (19909) Epidemiology (2067) Evolutionary Biology (24326) Genetics (15614) Genomics (22513) Immunology (17741) Microbiology (40423) Molecular Biology (17193) Neuroscience (88645) Paleontology (667) Pathology (2835) Pharmacology and Toxicology (4825) Physiology (7647) Plant Biology (15160) Scientific Communication and Education (2046) Synthetic Biology (4302) Systems Biology (9825) Zoology (2271)