OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials

OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials Ohad Landau , Kartheeswaran Thangathurai , Shai Magidi , Angel Porgador , Eitan Rubin doi: https://doi.org/10.1101/2025.08.08.669293 Ohad Landau 1 Ben Gurion University of the Negev , Beersheba, Israel Find this author on Google Scholar Find this author on PubMed Search for this author on this site For correspondence: olandau4{at}gmail.com Kartheeswaran Thangathurai 1 Ben Gurion University of the Negev , Beersheba, Israel 2 Department of Physical Science, University of Vavuniya , Sri Lanka Find this author on Google Scholar Find this author on PubMed Search for this author on this site Shai Magidi 3 Worldwide Innovative Network (WIN) Association - WIN Consortium , Villejuif, France Find this author on Google Scholar Find this author on PubMed Search for this author on this site Angel Porgador 1 Ben Gurion University of the Negev , Beersheba, Israel Find this author on Google Scholar Find this author on PubMed Search for this author on this site Eitan Rubin 1 Ben Gurion University of the Negev , Beersheba, Israel Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Preview PDF Abstract A critical challenge in personalized medicine is identifying the optimal drug treatment for individual patients based on their unique biological profiles. While recent advancements have led to a surge in drug prioritization algorithms utilizing patient omics data, including regression, classification, multiple kernel learning, deep learning, and AI-based methods, objective validation remains essential to discern their effectiveness. This study introduces” OptiRanker,” an innovative statistical framework designed to simulate and optimize the in vivo validation of these drug prioritization algorithms by ranking their predictive accuracy across varied conditions. OptiRanker not only mimics algorithmic predictions but also integrates them into ranked datasets, enabling the optimal selection of test subjects and drugs for trial setups. By providing a rigorous evaluation subset for ranking and comparison of high- and low-performing algorithms, OptiRanker facilitates the design of cost-effective in vivo trials tailored to efficiently assess predictive capability. This addresses a key bottleneck in personalized medicine: achieving a robust, scalable validation of computational models that can accelerate the clinical translation of promising algorithms. The efficient ranking provided by OptiRanker has the potential to advance personalized treatment strategies, ultimately leading to more effective, individualized patient care. The Python code for OptiRanker is available at the following GitHub repository: https://github.com/OhadLandau/OptiRanker . 1 Introduction The field of precision oncology has seen remarkable advancements in recent years, with the development of increasingly sophisticated diagnostic tools and targeted therapies tailored to the unique molecular profiles of individual tumours [ 1 ]. This paradigm shift has been driven by a deeper understanding of the complex genomic and immunological landscapes of cancer, which has enabled the development of therapies that target specific molecular alterations or other biological characteristics implicated in disease initiation and progression [ 2 – 4 ] yet the process of prioritizing and evaluating potential therapeutic candidates remains a significant challenge. [ 5 ] The rapid expansion of our knowledge about the molecular underpinnings of cancer has also revealed the inherent complexity and heterogeneity of these diseases, underscoring the need for individualized, gene-directed treatment approaches that go beyond the traditional tumor type-centric model [ 3 ]. In this context, the development of robust and reliable drug prioritization algorithms has emerged as a promising component of precision oncology, as clinicians and researchers seek to identify the most effective therapeutic strategies for each patient based on their unique characteristics [ 6 ]. However, the evaluation of these drug prioritization algorithms has been hampered by the inherent challenges of clinical trials, which often rely on small patient cohorts and may not capture the full spectrum of disease heterogeneity [ 2 ] or the high costs of such trials [ 7 ] and are further complicated by ethical considerations [ 8 ]. A further limitation in current clinical trial designs is the inability to test multiple drugs simultaneously on the same patient. This constraint is particularly problematic when various drug prioritization algorithms disagree on which therapeutic approach will elicit the best response. In such cases, only one drug can typically be tested in the patient at a time, preventing direct comparison and annotation of drug efficacy on a patient-specific basis. This limitation is especially relevant in the context of personalized medicine and when leveraging omics data. The field of drug prioritization algorithms has made significant strides in recent years. A wide range of computational approaches have been developed that leverage various types of data, including genomic, transcriptomic, and clinical information. Each method has its own strengths and limitations [ 9 ]. For example, some algorithms have been designed to prioritize drugs based on their ability to target specific genetic alterations or molecular pathways implicated in cancer, [ 10 , 11 ] while others have leveraged machine learning techniques to identify patterns in large-scale datasets that may inform treatment decisions [ 12 – 21 ]. Critically, the performance of these algorithms can vary significantly based on the data and methods used, as well as the clinical context in which they are applied [ 2 ]. Furthermore, performance metrics evaluated on diverse datasets may introduce biases and may not accurately reflect real-world clinical scenarios [ 22 ] or act as comparable benchmarks [ 23 ]. Although retrospective analyses and early-phase clinical trials have shown promise in matching targeted therapies with specific genomic aberrations, some researchers have argued that the use of personalized treatment approaches needs to be further validated before being ready for widespread clinical implementation [ 24 ]. As a result, there remains a pressing need to rigorously validate the performance of these algorithms and optimize the in vivo trials to evaluate their efficacy in real-world clinical settings [ 7 ]. A key challenge in evaluating drug prioritization algorithms lies in the difficulty of establishing a reliable ground truth for comparison [ 25 ]. In silico validation, while valuable, often relies on datasets that may not fully represent the complexities of real-world patient populations and treatment responses [ 26 ]. Retrospective analyses of clinical trials can be confounded by factors such as patient selection bias and variations in treatment protocols [ 27 ]. Prospective clinical trials specifically designed to validate these algorithms are ideal but can be prohibitively expensive and time consuming, particularly given the need to evaluate multiple algorithms and their performance across diverse cancer types and patient subgroups. The alternative to clinical trials is experimental in vivo models such as PDX mice, in which the barriers of clinical trials are much reduced or eliminated. It is possible, for example, to imagine a trial in which the same tumor is used to create multiple PDX mice and each is treated with the drugs selected by a different algorithm. Comparing the predicted efficacy to the observed response will help rank the prediction algorithms. However, with over 150 approved targeted drugs, and over 11,000 possible pairwise combinations, testing the performance of all is impractical. It is essential to choose a priori a smaller number of recommendations to test that will achieve a satisfactory ranking. To this end, we have developed “OptiRanker,” a simulation-based framework that aims to generate the ideal combination of drugs and patient numbers to determine the best-to-worst order of prioritization algorithms in a time- and cost-effective approach. The core premise of the OptiRanker is to provide a platform where a quantitative approach to assess the potential performance of drug prioritization algorithms can be done, allowing researchers to test the robustness and accuracy of these algorithms under a variety of conditions and to identify the optimal experimental design for in vivo validation studies. By simulating the performance of different algorithms researchers can identify the most informative and efficient experimental setups. This approach has the potential to significantly reduce the time and cost associated with ranking drug prioritization algorithms in preclinical and clinical trials, ultimately accelerating the translation of these powerful tools to clinical practice. 2 Methods 2.1 Data Sources 2.1.1 Simulated Data Initialization of Ground Truth A ground truth dataset was generated as a matrix of random permutations of integers ranging from 1 to the maximal number of drugs for each individual. This matrix represents the ideal or “ground truth” ranking of drugs for each individual (i.e., the outcome of a perfect drug prioritization algorithm). Generating a series of Predictors with increasing Noise Each predictor was generated by copying the ground truth dataset and introducing increasing levels of noise. The noise was introduced by randomly selecting individuals and performing a specified number of swaps within each individual’s drug rankings. For a given predictor P p : p individuals were randomly selected. For each selected individual, the rank of p random pairs of drugs was randomly swapped. This process was repeated for every individual, resulting in predictors with decreasing accuracy as the predictor index p increased. A new 2D matrix of ( drugs × individuals ) was created for each predictor. 2.1.2 Transcriptions Expression profiles from the Cancer Cell Line Encyclopedia (CCLE)[ 28 ] and drug response metrics (IC50, GDSC)[ 29 ] for a total of 386 drugs and 1405 cell lines were obtained. Data was downloaded from the CCLE website ( https://sites.broadinstitute.org/ccle/datasets , accessed on 16/11/2024 for both). Drugs and cell lines were filtered based on the following criteria: The drug was tested on all the chosen cell lines, therefore having experimentally determined IC50 values for all. The drug had data compatible with the selected algorithms; For example, Regorafenib—approved for metastatic colorectal cancer—was present in the WINTHER knowledge base but absent from DDPP. These criteria ensured a large number of cell lines and drugs were chosen while ensuring all have IC50 values. The result is a dataset of 37 drugs and 570 cell lines, which were used in the in-silico experiments ( Fig. 5 ). 2.2 DDPP, SIMS and WINTHER Three algorithms, developed in part by the authors, were chosen: DDPP[ 30 ], SIMS[ 11 ], and WINTHER[ 10 ], Each was used to prioritize drugs based on transcriptomic data. These algorithms integrate multi-omic data to personalize cancer treatment strategies, each utilizing different combinations of genomic, transcriptomic, and other molecular data to generate predictions. A complete description of these algorithms is beyond the scope of this paper and all a described elsewhere [ 10 , 11 , 30 ]. Briefly, WINTHER measures the match between transcriptomic data and literature on drug efficacy. SIMS focuses on identifying disturbed intervention points through multi-omic data (e.g., genomic sequencing, copy number variation, transcriptomics, and miRNA expression). It elucidates activation patterns of 24 cancer-related “interventional nodes” and recommends single or combination therapies based on a scoring system that ranks activated nodes for each patient. DDPP aims to predict the change in predicted survival duration for each targeted therapy by analyzing gene expression profiles associated with drug sensitivity pathways. It utilizes a comprehensive, tumor- and treatment-agnostic approach, leveraging transcriptomic data to generate continuous survival estimates rather than binary response predictions. All three algorithms require expression profiles from matched normal tissues. 2.3 Scoring a Predictor Each candidate predictor P p is scored against the ground-truth by a position-biased weighted mean-squared error (WMSE)[ 31 ]. Briefly, squared rank deviations of a given predictor form the ground truth are aggregated across all individuals and drugs. The error is weighted by a coefficient (β) so that mistakes involving highly ranked drugs incur a steeper penalty. Afterwards, aggregated raw errors are inverted and linearly rescaled to the unit interval, yielding an intuitive score where larger values denote better agreement with the ground truth. In index notation, WMSE for a predictor P defined over I individuals and D drugs is and are the ground truth and predicted ranks, respectively, for drug d in individual i ; i s the position weight assigned to that drug, controlled by the user selected discount factor β 2.4 Sub-setting To evaluate subsets of the data, the predictor rankings produced by these subsets were compared to rankings generated using the full dataset. The correlation between subset rankings and full-dataset rankings was calculated using the Spearman correlation coefficient [ 32 ]. In index notation, the standard Spearman correlation for two sets of ranks x and y of length M is: where x m and y m represent the corresponding rank values for the m -th element (e.g., drug or individual) in the subset and the full set, respectively, and and are their mean values. Standard Deviation for Drug Selection To identify the most informative drugs, the code sums individual-wise standard deviations (across predictors) for each drug: Where denotes the average of P p,i,d over all predictors. The d drugs with the largest σ d values are then selected . Standard Deviation for Individual Selection Similarly, for each individual i , the code computes per-drug standard deviations (across predictors) and sums them: These selected drugs and individuals were then used to create subsets with the highest variability, ensuring the most informative data was utilized in further analysis ( Fig. 2 ). 2.5 Cost-Effectiveness To further improve OptiRanker’s utility, a cost-effectiveness analysis was carried out. Each subset whose correlation ρ i,d (for a chosen number of individuals i and drugs d ) exceeded a user given threshold (see below) was considered. The subset’s “cost” is simply the sum of individuals and drugs, i + d . The cost-effectiveness measure (in percentage) is computed as: where ρ i,d is set to zero if the correlation is below the threshold. 3 Results 3.1 OptiRanker Overview As described in the methods section, we developed a method for simulating drug efficacy predictions with a known order of accuracy. We also describe how, given a ground truth, predictions can be ordered to capture their ability to predict drug ranking. The OptiRanker tool capitalizes on these capabilities to offer two modes of action ( Fig. 1 ). In the fully simulated mode, users make critical decisions on the design of a trial, deciding how many drugs and how many samples would be required for any user-defined accuracy. This is achieved by creating simulated data in which “predictors” with decreasing accuracy are created by adding noise to the ground truth and are ranked using a score that estimates their accuracy. The correlation between the predictor rankings obtained with all the drugs and samples and the ranking calculated with fewer drugs and samples is then calculated for decreasing numbers of drugs and samples (see below). In the ‘Ranking’ mode, experimental results are used as ground truth, and actual prediction algorithms are ranked by their ability to reconstruct the ground truth. Download figure Open in new tab Figure 1: Workflow of OptiRanker. Generalized flow of simulating and optimizing Drug Prioritization Algorithms ranking experiments. Ground truth is either fully simulated or given by real world data, in our case – IC50 of drugs for cell lines, given by GDSC [ 29 ] To test the quality of the simulation, we repeated it 100 times. For each generated dataset, an accuracy score was calculated for each predictor, and the results are summarized in Figure 2 . As expected, predictors created with more noise receive lower scores, validating both our simulation and the score. Download figure Open in new tab Figure 2: Data Generation and Noise Validation. A. Predictor scores were averaged across 100 simulations (individuals = 20, predictors = 20, drugs = 20), demonstrating the controlled noise induction and the ability of the simulation to correctly rank the predictors. B. Box plot of scores across all 100 simulations depicting the stochastic nature of the noise induction. Scores were MinMax scaled and inverted, as increased simulation size corresponded with greater noise and exponential scores. Download figure Open in new tab Figure 3: Informed Selection. A. Standard deviation of each drug, calculated across all individuals and all predictors. B. Standard deviation of each individual, calculated across all drugs and all predictors. Colours and height are proportional to standard deviation values. C, D. Predictor scores were recomputed on data subsets. In panel C all individuals were retained while the number of drugs was increased incrementally; in panel D all drugs were retained while the number of individuals was increased incrementally. For each subset size, two sampling strategies were compared: arbitrary selection and informed selection (based on the highest standard deviations in A or B). The resulting predictor rankings (best → worst) were Spearman-correlated with the ground-truth ranking. Simulations were performed with 100 iterations, 20 predictors, 20 drugs and 20 individuals, as described earlier. 3.2 Using Variance for Choosing Patients and Drugs Next, we examined the value of informed selections of drugs and individuals. Our analysis (see below) requires subsetting the data by incrementally increasing the number of drugs and/or individuals from 1 to the limit provided by the user. Drugs and individuals can be randomly (“arbitrarily”) chosen or taken from a list sorted by standard variation. We calculated the standard deviation across all individuals for each drug, across predictions ( Fig 2A ) and, reciprocally, across all drugs for each individual ( Fig 2B ) to differentiate predictors (see 2.4) and accurately reconstruct the ranked order while relying on fewer observations than the full set. Spearman’s ρ climbed rapidly for both the informed drug-driven subsets ( Fig 2C ) and the informed individual-driven subsets ( Fig 2D ). The results highlight the importance of informed selection to the final score and ranked order, as evidenced by the significant immediate increase in the Spearman correlation coefficient, thus also illustrating its importance in subset optimization. 3.3 Trial Optimization Trials can be optimized by choosing a subset of drugs and individuals to be tested. Fewer drugs and smaller trials require less resources but may not correctly rank the prediction algorithms. Using our simulated data, we can test the correlation obtained with all the drugs and individuals to the ranking obtained with subsets of drugs and individuals. The subsets of drug ranks within individuals for each simulated predictor were evaluated for their proximity to the ground truth results, and the correlation between the corresponding list of predictors ranked order and the known order when using all the dataset was then depicted ( Fig. 4A ). The strength of the correlation (r, Spearman correlation) between subsets increases as the increments become larger, the minimal strength specified by the user is highlighted to select the optimal subset for the investigator’s needs. As expected, predictor ranking was less accurate when fewer drugs and individuals were chosen. Nevertheless, very good accuracy (p>0.8) was observed with as few as 1, 2 or 3 individuals (with 5, 2 and 2 drugs, correspondingly). It should be noted here that these results are specific to the predictors we used, and a different number of drugs and individuals may be required to rank actual predictors. An interactive heatmap that allows the user to navigate through each subset and select the contents is then utilized for user-friendliness ( Fig. S2 ). We then generated a cost-effectiveness heatmap illustrating how incremental increases in the subset size affect the prediction accuracy compared to the total number of individuals and drugs ( Fig. 4B ). The resulting trend highlights the potential for optimizing in vivo trials. Download figure Open in new tab Figure 4: Heatmap of all possible subsets. A. Heatmap of the Spearman correlation coefficients between the ranked order of predictors generated from all possible informed subsets and the best-ranked order determined using the full dataset. The correlation values range from 0 (white), 0.5-0.8 (blue) and 0.8-1 (red), indicating the degree to which each subset can recapitulate the optimal ranking of predictors. Correlations above 0.8 (arbitrarily selected and user-defined) are highlighted. B. Cost-effectiveness heat map for all subset configurations that satisfy the specified correlation cut-off. Subsets whose correlation falls below the threshold are rendered uniformly in yellow, while those that meet or exceed it are colour-graded from light to deep red in proportion to their cost-effectiveness. Heatmap follows previous simulation structure: 20 drugs × 20 individuals × 20 predictors, repeated 100 times. 3.4 In-silico Trial To test the ability of the simulation to propose how many drugs and individuals are necessary to rank prediction algorithms, we used cell line data from the CCLE trials. The trial that generated the data is described in detail elsewhere [ 28 ]. Briefly, it involved determining the IC50 of many drugs for many cell lines. Efficacy predictions of three in-house algorithms (DDPP [ 30 ], WINTHER [ 10 ], and SIMS [ 11 ]) were compared to measured IC50, suggesting the following order for the algorithms: DDPP, SIMS, WINTHER ( Fig 5A ). Median expression of every gene across selected cell lines was used as its normal value. Then, reducing predictions of the three algorithms and CCLE ground truth results into a 2D space using PCA ( Fig 5B ) resulted in cluster distances corresponding to the predictors ranked order obtained by OptiRanker. The same ranked order (DDPP, SIMS, WINTHER) was reconstructed using only one drug and six individuals ( Supp Fig. S2 ). It is important to acknowledge that the right order of the three predictors may be achieved by chance. To address this, we tested this hypothesis by conducting 20 iterations of optimization using completely simulated data of the same dimensions – 3 predictors, 37 drugs and 570 individuals (i.e. cell lines). The integrity of the early minimal subsets was consistently recapitulated across iterations, adding strength and robustness to the optimized subset selection. This confirms the method we propose to reduce the number of drugs and individuals. Download figure Open in new tab Figure 5: In-silico trial. Three algorithms—WIN [ 10 ], SIMS [ 11 ], and DDPP [ 30 ]—were applied to predict drug responses using cell line transcriptomic data for 570 cell lines. Subsequently, 37 matching drugs were ranked based on their corresponding IC50 values for each. The drug response predictions generated by each algorithm were then compared to the ground truth IC50 results and ranked accordingly. A. Bar plots depicting algorithm scores. B. Predictions of the algorithms with the CCLE results clustered in a reduced dimensional space (PCA). 4 Discussion In this work, we present a novel simulation for ranking and an optimization tool for pre-clinical trials of drug prioritization algorithms, utilizing minimal resources to correctly rank the best-to-worst predictor, which, to the best of our knowledge, is the first of its kind. By transforming algorithm predictions into ranks of drugs, we were able to simulate a sequence of noisy predictors against tested drug responses. Then, we showed that subsets of the data was sufficient to recapitulate the simulated and known rank of predictors, given by Spearman coefficient to estimate the recapitulation accuracy. Finally, predictions from three distinct algorithms - WIN, SIMS, and DDPP - were then ranked against known drug response data, and the minimal subset of tests sufficient to recreate such ranking was selected to establish the proof of concept. The primary contribution of this work is the demonstration of the capability to objectively differentiate between drug prioritization algorithms, from strong to weak performers, by an experiment with optimal resource allocation and less then the entirety of data. We present “OptiRanker”, a framework that enables the design of efficient in vivo trials that will evaluate drug prioritization algorithms. A systematic and quantitative approach to evaluate the performance of drug prioritization algorithms is crucial for their future translation to clinical practice, and OptiRanker increases the feasibility of such an evaluation. Moreover, this approach is robust and can be used to assess seemingly different algorithms, often utilizing different modalities to prioritize drugs so long as they rank the same drugs. The validity of this framework is supported not only by extensive simulations exploring diverse predictions and drug rankings but also through in silico trial based on published algorithms and drug response databases. These validation steps demonstrate the reliability and applicability of the “OptiRanker” in real-world in vivo settings (even if not in clinical setting). Although our methodology has demonstrated effective results, certain limitations have been identified that will be addressed in future works, if feasible. Primarily, the approach of utilizing rank-based predictions from various algorithms does not account for the magnitude of difference between ranks, which could potentially lead to penalizing minor differences or erroneously rewarding substantial changes. While Weighted Mean Square Errors (WMSE) scoring mitigates the potentially erroneous nature of value to rank transitions, exponentially penalizing ranks might not capture real-world magnitudes of positional distances. This issue remains elusive and requires further research. Furthermore, a significant challenge emerges as the number of algorithms, drug candidates, and participant cohorts increases. This escalation necessitates larger sample sizes to derive meaningful Spearman correlation coefficients. However, this requirement may lead to impractical or unrealistic trial designs. Conversely, reducing the number of predictors can result in extremely small subsets of participants and drug candidates, potentially yielding high correlations by mere chance rather than true significance. This problem of contrast is a critical consideration in any in vivo study and is not unique to OptiRanker: how one balances statistical power with practical feasibility?. Finally, additional validation and benchmarking efforts are required to fully demonstrate the advantages of the proposed framework, which can be achieved through expanding the in silico trial size and transitioning to in vivo validation studies. Several studies have underscored the challenges associated with validating drug prioritization algorithms using traditional methods [ 33 , 34 ]. These challenges stem from the difficulty in establishing a reliable ground truth, the limitations of in silico datasets, and the high costs associated with prospective clinical trials. OptiRanker directly addresses the latter challenges by providing an efficient means to pre-clinically optimize in vivo trials. Future research will focus on addressing the limitations of the current framework, including identifying and incorporating a better and informed penalization of rank changes, exploring alternative correlation metrics for larger datasets, increasing in-silico trial size and integrating additional data modalities beyond transcriptomics. Moreover, collaborations with experimental groups will be actively pursued to validate the “OptiRanker” in real-world in vivo settings, further strengthening its translational potential. The development of such robust and efficient validation frameworks is essential for accelerating the deployment of drug prioritization algorithms. 5 Conclusion This work introduces the “OptiRanker”. By leveraging simulations and rank-based analyses, our approach enables the identification of minimal yet informative experimental designs. While further validation and refinements are necessary, the “OptiRanker” represents a step towards the efficient and cost-effective evaluation of drug prioritization algorithms, ultimately contributing to the advancement of personalized medicine. This aligns with the growing recognition in the field that robust and generalizable methods for evaluating computational approaches in drug discovery are crucial [ 35 ]. Additionally, the utilization of in silico simulations to guide experimental design, as showcased in this study, is gaining increasing traction in various areas of biomedical research [ 36 ], highlighting the timeliness and relevance of this work. Code and Data Availability All code and data used for the simulations, in-silico trial, scoring, and optimization processes presented in this paper are openly available on the GitHub below. This repository ensures reproducibility of the results and facilitates further research on the methodologies described. https://github.com/OhadLandau/OptiRanker Funding Sources OL was funded by ISF grant 2484/19. The Worldwide Innovative Network (WIN) Association - WIN consortium funded SM Competing Interest Authors have no competing interest to declare. Supplementary Information Download figure Open in new tab Supplementary S1 Snapshot from interactive heatmap generated by Plotly, displayed on local Dash app. Clicked cell displays the content of the selected subset. In this case, cell i4d1 with 0.81 correlation was selected. Download figure Open in new tab Supplementary S2 Snapshot of zoomed-in (10×10) interactive heatmap of In-Silico trial (full dimensions – 37 drugs X 570 cell lines X 3 predictors), with cost-effective analysis, generated by Plotly, displayed on local Dash app. Clicked cell displays the content of the selected subset. In this case, cell i6d1 with a correlation of 1 and cost-effectiveness of 14.29 Funder Information Declared Israel Science Foundation , 2484/19 Worldwide Innovative Network (WIN) Association References 1. ↵ Murciano-Goroff YR , Suehnholz SP , Drilon A , Chakravarty D. Precision Oncology: 2023 in Review . Cancer Discov . 2023 Dec 12; 13 ( 12 ): 2525 – 2531 . doi: 10.1158/2159-8290.CD-23-1194 . PMID: 38084089 ; PMCID: PMC10715685 . OpenUrl CrossRef PubMed 2. ↵ Shah , P. , Kendall , F. , Khozin , S. , Goosen , R. , Hu , J. , Laramie , J. , Ringel , M. , & Schork , N. J. ( 2019 ). Artificial intelligence and machine learning in clinical development: a translational perspective . npj Digital Medicine , 2 ( 1 ). Nature Portfolio . doi: 10.1038/s41746-019-0148-3 OpenUrl CrossRef PubMed 3. ↵ Tsimberidou , A. M. , Fountzilas , E. , Nikanjam , M. , & Kurzrock , R. ( 2020 ). Review of precision cancer medicine: Evolution of the treatment paradigm . Cancer Treatment Reviews , 86 . Elsevier BV . doi: 10.1016/j.ctrv.2020.102019 OpenUrl CrossRef PubMed 4. ↵ Tsimberidou , A. M. , Eggermont , A. M. M. , & Schilsky , R. L. ( 2014 ). Precision Cancer Medicine: The Future Is Now, Only Better . American Society of Clinical Oncology . doi: 10.14694/edbook_am.2014.34 . OpenUrl CrossRef 5. ↵ Blanco-González , A. , Pathirana , R. , Villanova , L. , Geris , L. , & Ramasamy , S. ( 2023 ). The Role of AI in Drug Discovery: Challenges, Opportunities, and Strategies . Pharmaceuticals , 16 ( 6 ). Multidisciplinary Digital Publishing Institute . doi: 10.3390/ph16060891 OpenUrl CrossRef PubMed 6. ↵ Adam , G. A. , Rampásek , L. , Safikhani , Z. , Smirnov , P. , Haibe-Kains , B. , & Goldenberg , A. ( 2020 ). Machine learning approaches to drug response prediction: challenges and recent progress . npj Precision Oncology , 4 ( 1 ). Nature Portfolio . doi: 10.1038/s41698-020-0122-1 OpenUrl CrossRef PubMed 7. ↵ Bentley , C. , Cressman , S. , van der Hoek , K. , Arts , K. E. , Dancey , J. , & Peacock , S. ( 2019 ). Conducting clinical trials—costs, impacts, and the value of clinical trials networks: A scoping review . Clinical Trials , 16 ( 2 ), 183 . SAGE Publishing . doi: 10.1177/1740774518820060 OpenUrl CrossRef PubMed 8. ↵ Emanuel EJ , Wendler D , Grady C. What makes clinical research ethical? JAMA . 2000 May 24-31; 283 ( 20 ): 2701 – 11 . doi: 10.1001/jama.283.20.2701 . PMID: 10819955 . OpenUrl CrossRef PubMed Web of Science 9. ↵ Ward , R. L. , & Ginsburg , G. S. ( 2017 ). Local and Global Challenges in the Clinical Implementation of Precision Medicine . In Elsevier eBooks . Elsevier BV . doi: 10.1016/b978-0-12-800681-8.00007-4 OpenUrl CrossRef 10. ↵ Rodon , J. , Soria , J.-C. , Berger , R. , Batist , G. , Tsimberidou , A. M. , Bresson , C. , … & Winther Consortium .( 2019 ). Genomic and transcriptomic profiling expands precision cancer medicine: the WINTHER trial . Nature Medicine , 25 ( 5 ), 751 . Nature Portfolio . doi: 10.1038/s41591-019-0424-4 OpenUrl CrossRef PubMed 11. ↵ Lazar , V. , Bibeau , F. , Karayan-Tapon , L. , De Forges , H. , Tabone-Eglinger , S. , Vacher , S. , … & Alifano , M. ( 2015 ). A simplified interventional mapping system (SIMS) for the selection of combinations of targeted treatments in non-small cell lung cancer . Oncotarget , 6 ( 16 ), 14139 . Impact Journals LLC . doi: 10.18632/oncotarget.3741 OpenUrl CrossRef PubMed 12. ↵ Deng , L. , Cai , Y. , Zhang , W. , Yang , W. , Gao , B. , & Liu , H. ( 2020 ). Pathway-Guided Deep Neural Network toward Interpretable and Predictive Modeling of Drug Sensitivity . Journal of Chemical Information and Modeling . American Chemical Society . doi: 10.1021/acs.jcim.0c00331 OpenUrl CrossRef 13. Nguyen , T. , Nguyen , G. T. T. , Nguyen , T. , & Le , D. ( 2021 ). Graph Convolutional Networks for Drug Response Prediction . IEEE/ACM Transactions on Computational Biology and Bioinformatics . doi: 10.1109/tcbb.2021.3060430 OpenUrl CrossRef 14. Liu , M. , Shen , X. , & Pan , W. ( 2022 ). Deep reinforcement learning for personalized treatment recommendation . Statistics in Medicine . Wiley . doi: 10.1002/sim.9491 OpenUrl CrossRef PubMed 15. Lee , K. , Bresson , C. , & Diaz , A. ( 2018 ). Deep learning of mutation-gene-drug relations from the literature . BMC Bioinformatics . BioMed Central . doi: 10.1186/s12859-018-2029-1 OpenUrl CrossRef 16. Lenhof , K. , Eckhart , L. , Gerstner , N. , Kehl , T. , & Lenhof , H. ( 2022 ). Simultaneous regression and classificationfor drug sensitivity prediction using an advanced random forest method . Scientific Reports , 12 ( 1 ). Nature Portfolio . doi: 10.1038/s41598-022-17609-x OpenUrl CrossRef 17. Rampásek , L. , Hidru , D. , Smirnov , P. , Haibe-Kains , B. , & Goldenberg , A. ( 2019 ). Dr.VAE: improving drug response prediction via modeling of drug perturbation effects . Bioinformatics . Oxford University Press . doi: 10.1093/bioinformatics/btz158 OpenUrl CrossRef PubMed 18. Lopez-Cortes , A. , Peralta , I. , & Marquet , M. ( 2024 ). Unraveling druggable cancer-driving proteins and targeted drugs using artificial intelligence and multi-omics analyses . Scientific Reports , 14 . Nature Portfolio . doi: 10.1038/s41598-024-68565-7 OpenUrl CrossRef PubMed 19. Piochi , L. F. , Preto , A. J. , & Moreira , I. S. ( 2023 ). DELFOS—drug efficacy leveraging forked and specialized networks—benchmarking scRNA-seq data in multi-omics-based prediction of cancer sensitivity . Bioinformatics , 39 ( 11 ). Oxford University Press . doi: 10.1093/bioinformatics/btad645 OpenUrl CrossRef 20. Zhang , F. , Wang , M. , Xi , J. , Yang , J. , & Li , A. ( 2018 ). A novel heterogeneous network-based method for drug response prediction in cancer cell lines . Scientific Reports , 8 ( 1 ). Nature Portfolio . doi: 10.1038/s41598-018-21622-4 OpenUrl CrossRef PubMed 21. ↵ Zhu , J. , Zheng , W. , & Zhao , Q. ( 2021 ). Prediction of drug efficacy from transcriptional profiles with deep learning . Nature Biotechnology , 39 ( 11 ), 1444 . Nature Portfolio . doi: 10.1038/s41587-021-00946-z OpenUrl CrossRef PubMed 22. ↵ Antoniou , T. , & Mamdani , M. ( 2021 ). Evaluation of machine learning solutions in medicine . CMAJ , 193 ( 36 ). Canadian Medical Association . doi: 10.1503/cmaj.210036 OpenUrl FREE Full Text 23. ↵ Hicks , S. A. , Zou , X. , Zhao , Y. , & Schilsky , R. ( 2022 ). On evaluation metrics for medical applications of artificial intelligence . Scientific Reports . Nature Portfolio . doi: 10.1038/s41598-022-09954-8 OpenUrl CrossRef PubMed 24. ↵ Cinti , C. , Trivella , M. G. , Joulie , M. , Ayoub , H. , & Frenzel , M. ( 2024 ). The Roadmap toward Personalized Medicine: Challenges and Opportunities . Journal of Personalized Medicine , 14 ( 6 ). Multidisciplinary Digital Publishing Institute . doi: 10.3390/jpm14060546 OpenUrl CrossRef PubMed 25. ↵ Kaptein , M. ( 2019 ). Personalization in biomedical-informatics: Methodological considerations and recommendations . Journal of Biomedical Informatics , 90 . Elsevier BV . doi: 10.1016/j.jbi.2018.12.002 OpenUrl CrossRef 26. ↵ Bajwa , J. , Munir , U. , Nori , A. V. , & Williams , B. ( 2021 ). Artificial intelligence in healthcare: transforming the practice of medicine . Future Healthcare Journal , 8 ( 1 ). Royal College of Physicians . doi: 10.7861/fhj.2021-0095 OpenUrl Abstract / FREE Full Text 27. ↵ Blonde , L. , Khunti , K. , Harris , S. B. , Meizinger , C. , & Skolnik , N. ( 2018 ). Interpretation and Impact of Real-World Clinical Data for the Practicing Clinician . Advances in Therapy , 35 ( 11 ). Springer Healthcare . doi: 10.1007/s12325-018-0805-y OpenUrl CrossRef PubMed 28. ↵ Ghandi , M. , Huang , F. W. , Jane-Valbuena , J. , Kryukov , G. V. , Lo , C. C. , McDonald , E. R. , … & Sellers , W. R. ( 2019 ). Next-generation characterization of the Cancer Cell Line Encyclopedia . Nature , 569 ( 7757 ), 503 . Nature Portfolio . doi: 10.1038/s41586-019-1186-3 OpenUrl CrossRef PubMed 29. ↵ Yang , W. , Soares , J. , Greninger , P. , Edelman , E. J. , Lightfoot , H. , Forbes , S. , … & Garnett , M. J. ( 2012 ). Genomics of Drug Sensitivity in Cancer (GDSC): a resource for therapeutic biomarker discovery in cancer cells . Nucleic Acids Research , 41 . Oxford University Press . doi: 10.1093/nar/gks1111 OpenUrl CrossRef PubMed Web of Science 30. ↵ Lazar , V. , Matei , F. , Tabone , S. E. , … & Alifano , M. ( 2021 ). Digital Display Precision Predictor: the prototype of a global biomarker model to guide treatments with targeted therapy and predict progression-free survival . npj Precision Oncology , 5 . Nature Portfolio . doi: 10.1038/s41698-021-00171-6 OpenUrl CrossRef 31. ↵ Yang , WC. , Ri , SC. , Ri , KS. et al. An improved GM(1,1) model based on weighted MSE and optimal weighted background value and its application . Sci Rep 14 , 30086 ( 2024 ). doi: 10.1038/s41598-024-81166-8 OpenUrl CrossRef PubMed 32. ↵ Schober P , Boer C , Schwarte LA . Correlation Coefficients: Appropriate Use and Interpretation . Anesth Analg . 2018 May ; 126 ( 5 ): 1763 – 1768 . doi: 10.1213/ANE.0000000000002864 . PMID: 29481436 . OpenUrl CrossRef PubMed 33. ↵ Failli , M. , Paananen , J. , & Fortino , V. ( 2020 ). Publisher Correction: Prioritizing target-disease associations with novel safety and efficacy scoring methods . Scientific Reports , 10 ( 1 ). Nature Portfolio . doi: 10.1038/s41598-020-58426-4 OpenUrl CrossRef 34. ↵ Ferreira FJN , Carneiro AS . AI-Driven Drug Discovery: A Comprehensive Review . ACS Omega . 2025 Jun 6; 10 ( 23 ): 23889 – 23903 . doi: 10.1021/acsomega.5c00549 . PMID: 40547666 ; PMCID: PMC12177741 . OpenUrl CrossRef PubMed 35. ↵ Partin , A. , Smith , M. , & Liu , Z. ( 2023 ). Deep learning methods for drug response prediction in cancer: Predominant and emerging trends . Frontiers in Medicine , 10 . Frontiers Media . doi: 10.3389/fmed.2023.1086097 OpenUrl CrossRef 36. ↵ Schneider , P. , Walters , W. P. , & Plowright , A. T. ( 2019 ). Rethinking drug design in the artificial intelligence era . Nature Reviews Drug Discovery , 19 ( 5 ). Nature Portfolio . doi: 10.1038/s41573-019-0050-3 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted August 12, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials Ohad Landau , Kartheeswaran Thangathurai , Shai Magidi , Angel Porgador , Eitan Rubin bioRxiv 2025.08.08.669293; doi: https://doi.org/10.1101/2025.08.08.669293 Share This Article: Copy Citation Tools OptiRanker: an Advanced Tool for Simulation and Comparative Analysis of Drug Prioritization Algorithms in in-Vivo Trials Ohad Landau , Kartheeswaran Thangathurai , Shai Magidi , Angel Porgador , Eitan Rubin bioRxiv 2025.08.08.669293; doi: https://doi.org/10.1101/2025.08.08.669293 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7633) Biochemistry (17680) Bioengineering (13889) Bioinformatics (41927) Biophysics (21445) Cancer Biology (18585) Cell Biology (25491) Clinical Trials (138) Developmental Biology (13373) Ecology (19897) Epidemiology (2067) Evolutionary Biology (24308) Genetics (15606) Genomics (22494) Immunology (17736) Microbiology (40385) Molecular Biology (17175) Neuroscience (88583) Paleontology (666) Pathology (2830) Pharmacology and Toxicology (4822) Physiology (7641) Plant Biology (15149) Scientific Communication and Education (2045) Synthetic Biology (4293) Systems Biology (9822) Zoology (2271)