Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells

Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells Owen M. O’Connor , View ORCID Profile Mary J. Dunlop doi: https://doi.org/10.1101/2024.07.11.603075 Owen M. O’Connor 1 Biomedical Engineering, Boston University , Boston, MA 02215, USA 2 Biological Design Center, Boston University , Boston, MA 02215, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Mary J. Dunlop 1 Biomedical Engineering, Boston University , Boston, MA 02215, USA 2 Biological Design Center, Boston University , Boston, MA 02215, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mary J. Dunlop For correspondence: mjdunlop{at}bu.edu Abstract Full Text Info/History Metrics Supplementary material Preview PDF Abstract Deep learning-based methods for identifying and tracking cells within microscopy images have revolutionized the speed and throughput of data analysis. These methods for analyzing biological and medical data have capitalized on advances from the broader computer vision field. However, cell tracking can present unique challenges, with frequent cell division events and the need to track many objects with similar visual appearances complicating analysis. Existing architectures developed for cell tracking based on convolutional neural networks (CNNs) have tended to fall short in managing the spatial and global contextual dependencies that are crucial for tracking cells. To overcome these limitations, we introduce Cell-TRACTR ( Tr ansformer with A ttention for C ell T racking and R ecognition), a novel deep learning model that uses a transformer-based architecture. The attention mechanism inherent in transformers facilitates long-range connections, effectively linking features across different spatial regions, which is critical for robust cell tracking. Cell-TRACTR operates in an end-to-end manner, simultaneously segmenting and tracking cells without the need for post-processing. Alongside this model, we introduce the Cell-HOTA metric, an extension of the Higher Order Tracking Accuracy (HOTA) metric that we adapted to assess cell division. Cell-HOTA differs from standard cell tracking metrics by offering a balanced and easily interpretable assessment of detection, association, and division accuracy. We test our Cell-TRACTR model on datasets of bacteria growing within a defined microfluidic geometry and mammalian cells growing freely in two dimensions. Our results demonstrate that Cell-TRACTR exhibits excellent performance in tracking and division accuracy compared to state-of-the-art algorithms, while also matching traditional benchmarks in detection accuracy. This work establishes a new framework for employing transformer-based models in cell segmentation and tracking. Author Summary Understanding the growth, movement, and gene expression dynamics of individual cells is critical for studies in a wide range of areas, from antibiotic resistance to cancer. Monitoring individual cells can reveal unique insights that are obscured by population averages. Although modern microscopy techniques have vastly improved researchers’ ability to collect data, tracking individual cells over time remains a challenge, particularly due to complexities such as cell division and non-linear cell movements. To address this, we developed a new transformer-based model called Cell-TRACTR that can segment and track single cells without the need for post-processing. The strength of the transformer architecture lies in its attention mechanism, which integrates global context. Attention makes this model particularly well suited for tracking cells across a sequence of images. In addition to the Cell-TRACTR model, we introduce a new metric, Cell-HOTA, to evaluate tracking algorithms in terms of detection, association, and division accuracy. The metric breaks down performance into sub-metrics, helping researchers pinpoint the strengths and weaknesses of their tracking algorithm. When compared to state-of-the-art algorithms, Cell-TRACTR meets or exceeds many current benchmarks, offering excellent potential as a new tool for the analysis of series of images with single-cell resolution. Introduction Single-cell resolution experiments have significantly enhanced our understanding of cell physiology and survival in the context of a wide range of domains including antibiotic resistance ( 1 ), cancer ( 2 ), stress response ( 3 , 4 ), and aging ( 5 , 6 ). Traditional population-level measurements can obscure heterogeneity among cells such as single-cell differences in gene expression that can emerge across space or time. Because this variation can be stochastic in origin ( 7 ), it is often necessary for researchers to gather large datasets to achieve statistically significant insights. Advances in automated imaging, coupled with increasing computer storage capacity, have dramatically increased the throughput of data collection. However, despite the abundance of data, analysis can remain a bottleneck, particularly for applications with many sequential images acquired over time. Although cell segmentation algorithms have achieved impressive performance levels ( 8 – 11 ), cell tracking still present a significant challenge. Cell tracking datasets are often markedly different from standard multi-object tracking datasets. In contrast to classic computer vision datasets like those that feature human subjects or cars moving through a field of view, cells can divide, and division events can occur frequently during the time frame of an experiment. Moreover, cells often display a high degree of similarity in appearance, are small in resolution, and can morph in shape over time. In addition, time-lapse microscopy experiments, which can extend for hours or even days, may have a low frame rate to reduce the impact of imaging on cells. However, the reduced frame rate can complicate tracking due to the non-linear motion often exhibited by cells. For example, growth of a microcolony may produce unpredictable motion of an individual cell, in contrast to images involving pedestrians or vehicle motion, which tend to follow more predictable patterns. In addition, shifts in the field of view can occur due to changes in imaging conditions over time. Further complicating analysis, microscopy images can contain a high density of cells, especially for applications that involve the analysis of cellular communication or community dynamics ( 12 , 13 ). Thus, cell tracking algorithms must contend with the complexities of cellular behavior coupled with the technical constraints of microscopy and live cell imaging. While early cell tracking models employed non-machine learning techniques ( 14 – 17 ), the landscape has shifted dramatically towards the adoption of deep learning based methods ( 18 ). The acceleration of AI in the past decade has ushered in a new era within the field of cell tracking, with deep learning models surpassing simpler machine learning approaches such as nearest-neighbor algorithms ( 19 ). Traditionally, most deep learning algorithms have employed a tracking-by-detection approach where segmentation and tracking are performed in two separate steps ( 14 , 20 – 22 ). However, a recent trend in the field is a move towards models where segmentation and tracking are performed in one step ( 23 – 29 ). Combining segmentation and tracking into one step can help ensure coherent predictions, where tracking information can enhance segmentation accuracy. For example, temporal information helps the segmentation model understand that cells can divide but not merge. Furthermore, some models now incorporate long-term temporal information, providing more context for segmentation ( 23 , 28 ). While these improvements have demonstrated superior performance compared to models that utilize the tracking-by-detection approach, these models typically still require separate decoders for segmentation and tracking ( 23 , 24 ) or rely on post-processing ( 27 , 28 ). A unified decoder could enhance the synchronization of segmentation and tracking, leading to more consistent predictions, while eliminating post-processing steps would simplify workflows and improve model generalizability. In cell tracking and segmentation software, convolutional neural networks (CNN) are currently the preferred method, evidenced by their widespread adoption within the field ( 10 , 23 – 25 , 27 , 28 ). In particular, U-Net ( 30 ) and its derivatives have been used extensively for cell tracking ( 10 , 20 , 25 , 26 , 28 ) and cell segmentation ( 10 , 31 – 33 ), demonstrating the versatility of this architecture. However, deep learning approaches in cell tracking also extend beyond CNNs. Other architectures, such as graph neural networks ( 34 , 35 ), reinforcement learning ( 36 ), recurrent neural networks ( 22 , 25 ), and unsupervised learning ( 19 ) approaches, have also been successfully employed. Despite these advances, the utilization of transformers ( 37 ), which have revolutionized many areas of machine learning including computer vision ( 38 ), remains relatively unexplored in cell tracking. Transformers use an attention mechanism that can effectively model the relationships between different parts of an image, making it easier to follow the same object across frames. This attention mechanism has demonstrated great success in cell segmentation, where transformers have made significant strides ( 9 , 11 , 39 ). While emerging work, like Trackastra ( 40 ), has demonstrated the promising potential of transformers in cell tracking, the significant capabilities of transformers, as demonstrated by the state-of-the-art multi-object tracking algorithm MOTRv3 ( 41 ), have not been fully leveraged for cell tracking. We were inspired by the top performance of MOTRv3 on the DanceTrack dataset ( 42 ), which features objects (dancers) with similar appearances and diverse motion patterns. This tracking dataset closely parallels cell tracking, where cells can have nearly identical appearances and migrate in non-linear directions. We reasoned that top performing algorithms on the DanceTrack dataset, like MOTRv3, would be well-suited to track cells. To leverage these capabilities, here we introduce a novel approach to cell segmentation and tracking that uses a transformer-based model to enhance tracking performance. Inspired by TrackFormer ( 43 ) and MOTR ( 44 ), which are multi-object tracking algorithms from the broader computer vision community, we developed a transformer-based model that can simultaneously segment and track dividing cells. This model builds upon the framework of the Detection Transformer (DETR) ( 45 ) which was the first deep learning model to perform end-to-end object detection without the need for any post-processing like non-maximum suppression. Although DETR demonstrates remarkable performance, it requires longer training periods and greater computational resources than CNN-based approaches. However, its advantage lies in the transformer’s ability to make long-range connections, integrating information across distant parts of the image. This advantage makes it uniquely equipped to handle issues that arise with cell tracking datasets, such as substantial shifts in the field of view or highly motile cells. The evolution of DETR has led to significant enhancements in terms of accuracy and reduced training times through innovations such as deformable attention ( 46 ), dynamic anchor boxes ( 47 ), contrastive denoised training ( 48 , 49 ), and query selection ( 50 , 51 ). Among these innovations, Mask-DINO ( 52 ) and Co-DETR ( 53 ) have emerged as state-of-the-art models for instance segmentation and object detection, respectively. Meanwhile, TrackFormer ( 43 ) and MOTR ( 44 ) have been instrumental in advancing tracking capabilities for DETR-based models. Recent iterations have built upon this foundation, offering further improvements ( 41 , 54 – 58 ), culminating in the latest iteration of MOTR, called MOTRv3 ( 41 ), which has established itself as a state-of-the-art model in multi-object tracking. Historically, cell tracking algorithms used datasets from the Cell Tracking Challenge ( 59 ) to benchmark performance. However, a notable portion of researchers have opted to use custom datasets or the increasingly popular DeepCell dataset ( 22 ). Although the Cell Tracking Challenge has provided a useful way to standardize evaluations, it has drawbacks since most of the datasets are limited in the number of images and are only partially annotated, posing challenges for deep learning models that can require large, fully annotated datasets. The DeepCell dataset addresses this by providing a large and diverse dataset complete with full annotations. In addition to providing datasets, the Cell Tracking Challenge also developed associated metrics to quantify cell segmentation and tracking accuracy. These metrics have been widely adopted for assessing model performance, however recent critiques, especially regarding their handling of cell division, have prompted researchers to modify these metrics or add new metrics ( 22 , 23 , 60 ). Similarly, we found that the tracking metric from the Cell Tracking Challenge tends to de-emphasize division accuracy while over-emphasizing segmentation precision. Meanwhile, the general multi-object tracking community has adopted the Higher Order Tracking Accuracy (HOTA) metric due to its balanced assessment between tracking and segmentation. HOTA offers interpretable results that can help identify where a model is under performing, however the classic version of this metric does not handle division events. In this work, we introduce Cell-TRACTR, a new transformer-based model to segment and track cells. The model can simultaneously segment and track without requiring any post-processing and it can handle frequent division events. Transformers offer a powerful architecture to handle the difficult challenge of tracking cells. In addition, we introduce a new cell tracking metric which adapts the HOTA metric ( 61 ) to effectively account for cell division. Cell-HOTA assesses overall tracking performance by balancing the algorithm’s performance on detection, association, and localization. This metric can be broken down into interpretable sub-metrics, providing useful insight into sources of error. We compare our new transformer-based model (Cell-TRACTR) to other state-of-the-art models in the cell tracking field (DeLTA and EmbedTrack). We evaluate the three models on both bacterial and mammalian datasets. Our results demonstrate Cell-TRACTR’s excellent performance on cell segmentation and tracking tasks and illustrate the interpretability of the Cell-HOTA metric. Results Cell-TRACTR simultaneously segments and tracks cells Our overall goal was to achieve accurate segmentation and tracking of cells given time-lapse microscopy data. We tested our algorithm on two types of data: microscopy images of Escherichia coli growing in the “mother machine,” ( 62 ) a widely used microfluidic device where bacteria are constrained to grow in single-file lines ( Fig 1A ), and mammalian cells with stained nuclei growing in a two-dimensional framework ( Fig 1B ) ( 22 ). These types of experiments can generate large amounts of data, motivating the need for automated image analysis. For example, the mother machine device contains thousands of parallel chambers, each of which is imaged over many hours. Cells growing in two dimensions present additional challenges, where they may move in any direction and morph in shape over time. Download figure Open in new tab Figure 1. Overview of Cell-TRACTR analyzing time-lapse microscopy movies. (A) Representative output from Cell-TRACTR for a single chamber from a time-lapse microscopy movie of E. coli bacteria growing in the mother machine microfluidic device. The output is displayed as a kymograph where the same chamber is shown over time. Each color represents a unique cell being tracked and black arrows signify cell divisions. This movie was taken from the test set. ( B) Overview of Cell-TRACTR tracking mammalian nuclei across multiple frames. Each color represents a unique cell being tracked. These frames were taken from the DeepCell dataset using data from the test set. (C) Cell-TRACTR is a DETR-based model that first uses a CNN backbone to extract image features, then uses a transformer encoder-decoder framework that leverages attention mechanisms. Encoded multi-scale features are used for query selection to initialize the object queries. Object queries attend to the encoded multi-scale features to generate the output embeddings needed to predict class labels, bounding boxes, and segmentation masks. If the model predicts an object query to be a cell, then the output embedding is shown as a colored box in the schematic. Object queries predicted to be of class “no object” are colored black and are discarded. The squares represent content embeddings and the circles represent positional embeddings. We developed Cell-TRACTR by employing a DETR-based framework that couples a CNN backbone with a transformer encoder-decoder architecture ( 45 ) ( Fig 1C ). This end-to-end model, where the input is the raw image and the output is embeddings that represent the position and lineage information for each cell, eliminates the need for post-processing techniques like non-maximum suppression that are required for other tracking by detection approaches ( 63 – 65 ). When an image is input into the model, we first use a CNN to preprocess the image into a more compact and rich representation since it would be computationally prohibitive to process the full image directly with a transformer. The goal of the CNN is to produce multi-scale features that can be fed into the transformer. Multi-scale features consist of the final layers that are output from the CNN, and they capture properties such as edges, colors, and simple textures. We adopted a ResNet18 ( 66 ) CNN architecture as the backbone to minimize model size while preserving high performance. Cell-TRACTR architecture: Cell segmentation After the CNN extracts information from the input image, an encoder further refines these features, preparing them for the decoder. The multi-scale features along with their positional embeddings are fed into the encoder. The multi-scale features serve as content embeddings which are latent vector spaces that store semantic information about the image. The positional embeddings are latent vector spaces that supply spatial context to the multi-scale features, compensating for the transformer’s inherent lack of spatial processing capability. In the encoder, deformable self-attention is used to further enhance the multi-scale features, following the approach in Deformable DETR ( 46 ). The model next performs “query selection”, also called “two stage” in Ref. ( 46 ), which aligns with the Deformable DETR framework, to convert the encoded multi-scale features into region proposals that are used to initialize the object queries (Fig S1, Supplementary Methods). Briefly, for each encoded multi-scale feature, a class label, bounding box, and segmentation mask prediction are generated. The class label represents the category of an object (‘cell’ or ‘no object’) while the bounding box represents the normalized center coordinates and the height, and width of the box with respect to the input image. The segmentation mask is an image which represents the pixels associated with each predicted cell. These masks are generated by integrating information from the output embeddings, the encoded multi-scale features, and the original multi-scale features; this process of mask generation is described in further detail below after tracking and cell division are introduced. The top-K encoded multi-scale features are selected based on the highest values of the class labels (Fig S1). Bounding boxes, extracted from the top-K segmentation masks, are used as the positional embeddings and the top-K encoded multi-scale features are used as the content embeddings. K is determined by the number of the object queries. Combining these positional and content embeddings forms the object queries which are passed to the decoder. Within the decoder, the object queries interact with the encoded multi-scale features to generate output embeddings that are used to refine an object’s bounding box and predict the segmentation mask. The object queries first perform self-attention to avoid duplicate predictions and then deformable cross-attention, where the object queries spatially attend to the encoded multi-scale features (Fig S1). Output embeddings that are not predicted to be cells (i.e. their class label is ‘no object’) are discarded. Thus, the final output of the decoder is a set of output embeddings, where each embedding encodes information about the class label, location of each predicted cell within the image given by both a bounding box that defines its location and a segmentation mask that defines the pixels associated with the cell. Cell-TRACTR architecture: Cell tracking We next leveraged the output embeddings to track cells from frame to frame using the same network architecture. To demonstrate this, we developed a model that tracks cells over multiple frames using a transformer architecture ( Fig 2A ). We based the tracking scheme on the architecture of TrackFormer ( 43 ) and MOTR ( 44 ), which are DETR derivatives, where output embeddings are converted into track queries for analysis of the subsequent movie frame ( Fig 2B ). At time t 0 , the model performs object detection and locates all cells in the image. For all subsequent frames, the output embeddings associated with each detected cell from the previous frame serve as track queries. Track queries predicted as ‘no object’, which represent cells that exited the field of view, are discarded. Track queries that are predicted to be cells are then concatenated with the object queries of the present frame and fed to the decoder. Self-attention is a critical step in the decoder where track and object queries need to communicate so that an object query does not detect a cell that is already being tracked (Fig S2). For example, when a cell enters the field the view, an object query is needed to detect the new cell. However, if a cell existed in the previous frame, it needs to be handled by a track query and not an object query. In addition, if a track query makes an error and misses a cell, an object query is needed to detect the missed cell in the next frame. In these scenarios, it is important that the object and track queries can communicate to avoid duplicate predictions. By utilizing object and track queries, Cell-TRACTR simultaneously segments and tracks cells, leading to more accurate and coherent predictions. Download figure Open in new tab Figure 2. Visual representation of how Cell-TRACTR uses track queries to track cells and detect divisions. (A) Overview of Cell-TRACTR performing cell tracking, where an arrow connecting two cells signifies a cell division event, as seen in frame t 2 . (B) In frame t 0 , the model performs object detection and locates all cells in the frame. In subsequent frames, the output embeddings serve as track queries, which track the locations of the cells from frame to frame. An output embedding split with two colors indicates that the model predicted a cell division. A black output embedding indicates when a cell, represented by a track query, is lost, such as when it exits the field of view. (C) Each output embedding makes two predictions. Each prediction can be classified as either as a cell or no object. The class labels determine which prediction is used. If only the first prediction for an output embedding is classified as a cell, then just the first prediction is used. We ignore predictions from output embeddings that predict no object for both class labels. When an output embedding is labeled as a division, both predictions are classified as cells. Cell-TRACTR architecture: Cell division Cell division is an important property that needs to be tracked accurately in single-cell imaging experiments. When a division occurs, as shown in the frame at time t 2 , the output embedding must record a division event, and this is represented schematically by a split light/dark blue output embedding ( Fig 2B ). For each output embedding, the model makes a class label, bounding box, and segmentation mask prediction ( Fig 2C ). For each of these tasks, two predictions are generated. The first prediction corresponds to the cell that is being detected or tracked. The second prediction corresponds to a potential daughter cell generated in a division event, which is only applicable for track queries since object queries can detect at most one cell. The classification branch uses a linear layer to classify each output embedding as ‘cell’ or ‘no object’ (Fig S3). An output embedding is classified as a ‘division’ when both outputs are classified as ‘cell’. The object detection branch uses a multi-layer perception (MLP) to generate a bounding box prediction for each cell. In addition, the segmentation branch produces a segmentation mask for each cell. Closely following the architecture of Mask-DINO ( 52 ), multi-scale features are combined with the largest encoded multi-scale feature map to form a pixel embedding map (Fig S3). The dot product between each output embedding and the pixel embedding map is used to generate the segmentation masks. This process assigns a value ranging from 0 to 1 to every pixel within the mask, indicating the likelihood of each pixel being part of a segmented cell. The maximum value attained by any pixel across all generated masks is combined into one unified mask that encapsulates all predictions. Pixels with values greater than 0.5 in this final mask are classified as belonging to a segmented object. If an output embedding is not predicted to be a cell (i.e. it has a class label of ‘no object’), then the associated bounding boxes and segmentation masks are disregarded. Using a transformer-based model allows for seamless segmentation and tracking of cells in a unified step. As an example, we used Cell-TRACTR to process a time-lapse microscopy movie of E. coli growing in the mother machine device (S1 Movie). In the first frame, the object queries detect all cells in the chamber. Then, these object queries are converted into track queries and tracked throughout the rest of the movie. These data provide an example demonstrating that Cell-TRACTR can accurately detect cells, track cells, and predict cell divisions over the course of many generations of growth. Multi-object tracking and cell tracking metrics To assess the performance of Cell-TRACTR, we developed an extended version of the Higher Order Tracking Accuracy (HOTA) metric ( 61 ). Multi-object tracking requires the detection and association of several objects over consecutive frames. When evaluating multi-object tracking, it is important to consider a model’s performance across a range of tasks including the detection, association, and localization of objects. Various tracking metrics have been proposed, but many metrics like Multiple Object Tracking Accuracy (MOTA) ( 67 ) and ID Switches (IDS) ( 68 ) have been shown to disproportionately emphasize either detection or association ( 61 , 69 ). An ideal tracking metric should value both detection and association, providing an accurate assessment of the overall performance. Recently, Higher Order Tracking Accuracy (HOTA) ( 61 ) has emerged as a leading metric for assessing tracking accuracy due to its balanced approach towards assessing detection, association, and localization. However, most multi-object tracking metrics from computer vision, including HOTA, are designed for objects that do not divide, which is a major limitation for cell tracking applications where division events can be common. In this work, we extended the HOTA metric to include division in its evaluation. Other approaches exist for measuring the overall performance of cell tracking algorithms. For example, the Cell Tracking Challenge developed a widely used metric, OP CTB (overall performance for the cell tracking benchmark). OP CTB is calculated as the mean of the tracking accuracy measure (TRA) and segmentation accuracy measure (SEG), 𝑂𝑃 !"# = 0.5 ∗ (𝑇𝑅𝐴 + 𝑆𝐸𝐺), where tracking is assessed using a graph matching-based approach and segmentation is assessed using the Jaccard Similarity index (Supplementary Text). Although OP CTB is widely used, recently it has been criticized for not properly handling cell division ( 22 , 23 , 60 ). In some cases, removing division links can increase the tracking score. For example, a division event predicted one frame early or late with respect to the ground truth can result in a lower score than removing the division all together (Supplementary Text, Fig S4). This is a problem because in real microscopy data it can be challenging to pinpoint the precise frame when a cell divides, and this issue is exacerbated in the case of rapidly growing cells where there are many division events ( Fig 3A ). Download figure Open in new tab Figure 3. Cell-HOTA metric incorporates cell division accuracy. (A) Kymograph of bacteria growing in the mother machine. Colored triangles indicate potential times at which a cell division could have occurred, where different colors correspond to different cells. Multiple triangles of the same color indicate that it can be subjective to determine where division occurs. (B) The intersection over union (IOU) between a tracker cell and ground truth cell must exceed the similarity threshold α to be considered a match. HOTA is evaluated across various values of α, with a higher α imposing stricter criteria for matching. (C) Schematic showing the criteria required to determine if a division qualifies as a true positive division (TPD). A tracker cell and ground truth cell are shown dividing at time t 1 . First, the set of daughter cells that divided at time t 1 needs to match. Second, the parent cells of those daughter cells at time t 0 needs to match. Third, the IOU between the set of daughter cells needs to exceed the similarity threshold α. Red lines indicate matching cells. The blue dotted line indicates the set of cells used to calculate the IOU to determine if the division is a TPD. (D) An early or late division occurs when a cell division is predicted one frame too early or late. Black lines indicate temporal associations and brown lines are used to highlight division events. (E) DivA rewards division events within a frame of the ground truth. Flexible divisions may be disabled so that divisions events are only counted as correct if they occur in the same frame as the ground truth. No Flex indicates that DivA has disabled flexible divisions. Cell-HOTA metric We sought to take advantage of the benefits of the HOTA metric—namely its ability to assess performance in a balanced manner across a range of segmentation and tracking associated tasks—while extending it to handle cell division events. Further, because HOTA is the emerging standard in the broader multi-object tracking community, we wished to align the cell tracking community more closely with this larger group. The HOTA metric is composed of two main sub-metrics for evaluating tracking: detection accuracy (DetA) and association accuracy (AssA). DetA measures detection accuracy by comparing the number of correct object detections to the total number of detections made. AssA evaluates how closely the predicted paths of objects match their actual paths, averaged over the entire set of correctly detected objects. At a high level, DetA is focused on assessing segmentation accuracy and AssA is specifically tailored to measure tracking accuracy. HOTA incorporates localization accuracy into the score by measuring DetA and AssA across varying similarity thresholds α ( Fig 3B ). The intersection over union (IOU) between a predicted object and a ground truth object needs to exceed α for the two objects to be considered a match, thus the α value determines the stringency for these matches. HOTA is measured at various values of α to provide insight into how stringency in matching affects DetA and AssA. To improve the readability and follow the terminology used in HOTA, we define a ‘tracker cell’ as a cell predicted by the tracking algorithm and a ‘ground truth cell’ as a cell defined in the ground truth. There are several advantages that HOTA offers compared to OP CTB (Supplementary Text). First, HOTA is a comprehensive standalone metric that can rank tracking algorithms and can also be decomposed into sub-metrics, allowing for straightforward analysis of performance. Second, HOTA integrates localization accuracy by calculating the average score across all HOTA α scores for each value of α. This is important because it allows the user to assess the detection and tracking performance at the level of stringency that is appropriate for their application. Third, HOTA functions as a monotonic metric with DetA and AssA, meaning an improvement in DetA or AssA will always increase the final HOTA score. Overall, HOTA presents a robust framework for evaluating and ranking tracking algorithms. To adapt HOTA for cell tracking, we introduced a metric for division accuracy (DivA) alongside the scores for detection and association accuracy. We call this modified metric Cell-HOTA. DivA α is the mean division accuracy score across all cell divisions at a similarity threshold α . A true positive division (TPD) is defined by three criteria ( Fig 3C ): First, two divided tracker cells with the same parent cell need to match with two divided ground truth cells with that same parent cell. Second, the parents of the divided tracker and ground truth cells need to match in the previous frame. Finally, the average IOU between the two matches needs to be greater than α . A false positive division (FPD) occurs when a tracker division does not match with a ground truth division. A false negative (FND) division occurs when a ground truth division does not match with a tracker cell division. Due to the importance of tracking cell divisions in time-lapse microscopy experiments, we set DivA α to be of equal importance to the association score AssA α . AssDivA α is the geometric mean of AssA α and DivA α whereas we defined Cell-HOTA α to be equal to the geometric mean of DetA α and AssDivA α . Similarly to HOTA, Cell-HOTA is a calculated by taking the average Cell-HOTA α score across all values of α. The Cell-HOTA metric is a single unified score that can also break down into multiple informative sub-metrics. This makes it easy to analyze whether the algorithm had issues detecting cells, tracking cells, or predicting divisions. Allowing for flexibility in determining time of cell division Another key consideration for cell tracking metrics is that the ground truth is often a subjective assessment of the cell division time rather than an unequivocal record of this event ( Fig 3A ). For cells that divide frequently, such as in time-lapse images of bacteria, these issues can compound and result in inaccurate estimates of tracking performance. Ultimately, we want to measure a model’s capability to generate a coherent cell track rather than replicate the precise ground truths. There are two possible scenarios associated with data where the precise division timing is uncertain and the ground truth is a close, but not exact, record of cell division. We define these alternatives as ‘early division’ and ‘late division’ ( Fig 3D ). In early division, the tracking algorithm predicts that a division event occurs before it does in the ground truth, while in late division it occurs after. In both cases, the overall lineage is preserved outside of this time window. To accommodate the flexible timing of divisions, we modified the DetA α , AssA α , and DivA α to handle early or late divisions that were one frame away from the ground truth ( Fig 3E ). In our analysis, we assume any division event within a frame of the ground truth is acceptable, while any division event more than a frame away from the ground truth is erroneous. The flexible division feature within Cell-HOTA is optional and may be turned off. More details on how Cell-HOTA handles flexible divisions are provided in the Methods. Benchmarking Cell-TRACTR and other algorithms To benchmark the performance of Cell-TRACTR, we compared it to two other deep learning-based models and analyzed performance on the same datasets. The first model we benchmarked against was the Deep Learning for Time-Lapse Analysis (DeLTA) ( 10 ) algorithm we developed previously. DeLTA consists of two CNNs based on the U-Net ( 30 ) architecture, where the two CNNs are used sequentially for segmentation and tracking. DeLTA is a popular model for the analysis of single-cell time-lapse movies of growing bacteria. The second model we benchmarked against was EmbedTrack ( 24 ), which is a CNN consisting of a shared encoder, two decoders for segmentation, and one decoder for tracking. We chose EmbedTrack due to its high performance on the Cell Tracking Challenge. Analyzing bacterial mother machine movies We first trained the three models (Cell-TRACTR, DeLTA, EmbedTrack) on a dataset consisting of 73 time-lapse microscopy movies of bacteria growing in the mother machine. The dataset was split 80%/20% into training and validation data. We trained each model on the training set and then evaluated it on a separate test set which consisted of 29 time-lapse microscopy movies. When analyzing Cell-HOTA α at various similarity thresholds α, we found that all three algorithms performed well on the dataset, especially at lower values of α. However, there were nuanced differences in performance, with Cell-TRACTR exhibiting superior performance for all α < 0.80 ( Fig 4A ). Above this threshold, EmbedTrack and DeLTA’s performance slightly exceed Cell-TRACTR, but all dropped off rapidly at high values of α. This rapid decrease is expected given the stringency of the similarity threshold in this regime. Overall, we found that all models exhibit similar performance, with Cell-TRACTR surpassing EmbedTrack and DeLTA in overall tracking performance, albeit with slightly less localization precision. Download figure Open in new tab Figure 4. Evaluation of model performance on the bacterial mother machine dataset. (A) Cell-HOTA evaluated at various similarity thresholds α on the bacterial mother machine dataset, using data from the test set. (B) Cell-HOTA 0.5 results on the test set. (C) Overall results on the test set. Cell-HOTA has three sub-metrics: DetA, AssA, and DivA. OP CTB has 2 sub-metrics: SEG and TRA. (D) Cell-HOTA score with and without flexible divisions. (E) Comparison of model efficiency. FPS is the inference time measured in frames per second, so higher values correspond to faster inference times. Params is the number of parameters in each model. M, million. Training time is the number of hours needed to train each model. To gain a comprehensive understanding of model performance, we next looked at the sub-metrics of Cell-HOTA. To report a single number, it is common to see the metrics evaluated either at a specific value of α or averaged across all values of α. We first selected a single α, choosing α = 0.5 because it is a typical value used in the literature ( 52 , 70 ). Cell-TRACTR had the highest overall tracking performance as indicated by Cell-HOTA 0.5 ( Fig 4B ). Decomposing this into sub-metrics, we found that all three models have comparable performance for DetA 0.5 , with EmbedTrack slightly outperforming Cell-TRACTR and DeLTA. In contrast, Cell-TRACTR excels in tracking and cell division, scoring the highest for AssA 0.5 and DivA 0.5 at levels that exceed the performance of both DeLTA and EmbedTrack. This indicates that while all models excel in detection, Cell-TRACTR is superior in tracking and division, while EmbedTrack has the most precise segmentation. We also looked at the overall Cell-HOTA score, which integrates across all values of α, and compared it to the Cell Tracking Challenge OP CTB score. With Cell-HOTA, we found that Cell-TRACTR and EmbedTrack exhibited closely matched performance, while DeLTA performed slightly less well ( Fig 4C ). When using the OP CTB metric, EmbedTrack exhibited slightly higher overall performance across the three algorithms due to its relatively high SEG score. The TRA scores for all three models showed minimal variation, while there were differences in the SEG scores. This underscores the excessive weight given to the SEG score in OP CTB . In addition, the TRA score offers limited insight into the model’s actual detection and tracking performance (Supplementary Text). Overall, we found that the Cell-HOTA score offers an interpretable and balanced method for assessing model performance. To understand the impact of permitting early and late divisions in Cell-HOTA, we evaluated the performance with and without flexible divisions ( Fig 4D ). All scores dropped significantly when flexible divisions were not allowed. Most notably, DivA consistently had the largest drop, demonstrating that all models struggled with pinpointing the exact frame a cell divides. It is clear that early and late divisions have a significant impact on the assessment of model performance. As the ground truth is often a subjective assessment of cell division time rather than the absolute truth, providing some leniency is likely to be critical for applications like bacterial cell tracking that exhibit many division events. We also evaluated the time needed to process the test dataset in terms of frames per second ( Fig 4E ). The more frames a model can evaluate per second, the faster the model, thus larger values correspond to faster inference. Cell-TRACTR had the fastest inference, with DeLTA performing second fastest. Although EmbedTrack had the slowest inference, its model utilized the least number of parameters and required the least amount of time to train. DeLTA utilized the most parameters and required the most time to train because it uses a separate model for segmentation and tracking whereas Cell-TRACTR and EmbedTrack only use one model. Analyzing mammalian DeepCell microscopy movies We next asked whether Cell-TRACTR would perform well on a very different cell segmentation and tracking task. For this, we considered data from the DynamicNuclearNet Tracking dataset from DeepCell ( 22 ). In this dataset, five mammalian cells lines were cultured on 96-well plates. Nuclear staining was performed to assist with cell identification. In this two-dimensional dataset, the mammalian cells can grow in any direction, in contrast to bacterial growth in the mother machine which is constrained to a single direction. Using the Cell-HOTA tracking metric, we evaluated the performance of Cell-TRACTR, DeLTA, and EmbedTrack on the DeepCell dataset. Cell-TRACTR demonstrated the highest performance for α < 0.80 while EmbedTrack was superior for higher α values ( Fig 5A ). As before, we evaluated Cell-HOTA at α = 0.5 ( Fig 5B ) and using the integrated Cell-HOTA score ( Fig 5C ). In both cases, we found Cell-TRACTR to be the top performer due to its balanced performance across the three sub-metrics ( Fig 5C ). For the overall Cell-HOTA score ( Fig 5C ) we found that EmbedTrack had the highest DetA and AssA scores due to high precision in segmentation, however its DivA score dramatically lowered its overall Cell-HOTA performance, ultimately impacting its overall performance. The OP CTB metrics confirm EmbedTrack’s overall quality in segmentation precision (SEG score), and EmbedTrack also achieved the top OP CTB TRA score (Supplementary Text). Overall, the results from the DeepCell dataset suggest that Cell-TRACTR offers excellent detection, tracking, and division accuracy (S2 Movie). Furthermore, the algorithm is able to handle diverse cell segmentation and tracking tasks, as evidenced by its strong performance on both bacterial and mammalian time-lapse microscopy data. Download figure Open in new tab Figure 5. Evaluation of model performance on the mammalian DeepCell dataset. (A) Cell-HOTA evaluated at various similarity thresholds α on the mammalian DeepCell dataset, using data from the test set. (B) Cell-HOTA 0.5 results on the test set. (C) Overall results on the test set. Cell-HOTA has three sub-metrics: DetA, AssA, and DivA. OP CTB has 2 sub-metrics: SEG and TRA. (D) Comparison of model efficiency. FPS is the inference time measured in frames per second, so higher values correspond to faster inference times. Params is the number of parameters in each model. M, million. Training time is the number of hours needed to train each model. When analyzing inference times, we found that EmbedTrack had the fastest inference. Cell-TRACTR and DeLTA had lower inference times due to the larger image sizes and increased number of cells. Similar to the bacterial mother machine dataset, EmbedTrack utilized the least number of parameters and required the smallest training time. While Cell-TRACTR needed far fewer parameters than DeLTA, both required extensive training times. Although training times for Cell-TRACTR are currently long when image sizes are large, as we discuss below, recent improvements in computational efficiency for DETR-based algorithms could address this issue. Discussion In this work, we developed a new transformer-based architecture, Cell-TRACTR, and showed that it excels at segmenting and tracking cells within time-lapse microscopy movies, exceeding or matching the performance of current state-of-the-art software. This architecture, based on TrackFormer ( 43 ), can track cells in an end-to-end manner without any post-processing. Cell-TRACTR uses track queries to take advantage of a cell’s past history to make a coherent prediction in the subsequent frame. These track queries also allow for segmentation and tracking to be accomplished in one unified step which is important for ensuring high accuracy. Although other deep learning models, like EmbedTrack ( 24 ) and DistNet2D ( 23 ), can simultaneously segment and track cells, they typically require multiple decoders, whereas Cell-TRACTR uses a single decoder. This helps ensure that segmentation and tracking predictions are in sync across subsequent frames because the use of separate decoders for segmentation and tracking can lead to discrepancies in the predictions. A future advantage of this DETR-based architecture is that it could allow Cell-TRACTR to handle objects that are occluded or disappear. For example, Cell-TRACTR could discard track queries only if they are classified as ‘no object’ for multiple frames, as in TrackFormer ( 43 ). To the best of our knowledge, Cell-TRACTR is the first algorithm to simultaneously segment and track cells with a single decoder while requiring no post-processing. To assess algorithm performance, we extended the widely-used HOTA metric to incorporate cell division, creating the Cell-HOTA metric. We also calculated performance using the traditional Cell Tracking Challenge OP CTB metric as a point of comparison. Cell-HOTA can be broken down into detection, association, and division accuracy, all of which are calculated across a range of similarity thresholds, α. We found that Cell-TRACTR excelled in association and division accuracy while all three models demonstrated comparable detection capabilities. Overall, Cell-HOTA enables the assessment of detection, association, and division, offering crucial insights for evaluating algorithm performance in cell tracking applications. We found that Cell-TRACTR delivered strong performance on both a bacterial mother machine dataset and a mammalian dataset from DeepCell. However, we did encounter computational limits when working with large two-dimensional datasets. For example, training times for two-dimensional bacterial images containing hundreds of cells in 584x600 pixel images exceeded a week, making hyperparameter optimization infeasible, leading us to conclude that this goal was impractical with the current version of Cell-TRACTR. A well-documented limitation of transformers is their computational complexity, a challenge that can be pronounced with microscopy data, where image sizes can be large and contain many cells at low resolution. The need to process large images leads to large multi-scale feature maps, while the low resolution of the cells necessitates an increase in the number of multi-scale feature maps, making this a computationally demanding task. However, there is recent promising work focused on making DETR-based models more computationally efficient, which offers future potential for this area. Examples include RT-DETR ( 71 ), D 2 ETR ( 72 ), and Efficient DETR ( 51 ), which increase computational efficiency, while SOF-DETR ( 73 ) and work by Huang et al . ( 74 ) offer approaches that are better at handling small objects. These new models mainly focus on engineering the encoder to be more efficient. Building upon the momentum for increased computational efficiency, DEFA ( 75 ) proposes a more efficient deformable attention that significantly reduces memory usage and training time. Departing from the tracking-by-query paradigm, MOTIP ( 58 ) utilizes a DETR-based model to generate object embeddings for each time frame which are used to address an ID prediction problem similar to Trackastra ( 40 ). Given the recent focus on improving the computational efficiency of DETR-like algorithms, we anticipate that future advances in this space will likely address some of the limitations we encountered. Integrating these improvements into Cell-TRACTR could enable cell segmentation and tracking of larger and increasingly complex time-lapse images. Another computational challenge is that DETR-based models that perform segmentation require a large amount of memory, as each output embedding produces a segmentation mask of the whole image. This is computationally demanding, for example when there are 250 cells in an image, 250 full-sized segmentation masks will be produced. When working with large two-dimensional images, we sometimes encountered CUDA out-of-memory errors during backpropagation for the segmentation masks. Future efforts could focus on only backpropagating the loss through sampled regions within the segmentation mask to reduce the computational load. Recently, Frozen Mask-DETR ( 70 ) developed a computationally efficient way to segment images with DETR. The model uses DINO-DETR ( 49 ) to predict bounding boxes, which are used as regions of interest that are segmented by an Instance Mask Decoder. This method significantly reduces the computational load by only segmenting the area within the bounding box while also still outperforming Mask-DINO ( 52 ) in instance segmentation. Similar to our work, Trackastra developed a computationally efficient way to track cells utilizing transformers ( 40 ). Their method consists of feeding object positions and shallow object features to a transformer and outputting an association matrix. Shallow object features include characteristics such as cell area and cell shape. The association matrix details how cells are related between two adjacent frames. Different from Trackastra, our work utilizes a tracking-by-query paradigm, can segment cells in addition to tracking cells, and requires no post-processing. Trackastra is a nice complement to our work demonstrating a lightweight transformer-based model for cell tracking. Cell-TRACTR currently processes one frame at a time, whereas other cell tracking algorithms analyze multiple frames at a time to incorporate temporal information into their models ( 22 , 23 , 28 ) and this is a potential extension of this work. Specifically, in Cell-TRACTR, track queries only attend to image features in the current frame of interest, however, they could attend to the previous and future frames for additional context. This could be achieved by stacking multi-scale features from a sequence of frames and feeding them to the encoder, as in TrackFormer ( 43 ). However, the feasibility of this approach is contingent on significant improvements in the computational efficiency of the encoder and decoder. Another extension of this work could include using the output embeddings for other auxiliary tasks like performing classification based on cell morphology. For such tasks, the weights of Cell-TRACTR could be frozen to ensure the segmentation and tracking performance remain high while minimizing the computational load. The Cell-HOTA metric could also be extended in several ways. We designed Cell-HOTA to work well on time-lapse images of cells that divide frequently. As implemented, Cell-HOTA weighs DivA and AssA equally, which places a high emphasis on division accuracy. For other use cases where cell division is less common, it may be more appropriate to adjust the metric to de-emphasize DivA relative to the other accuracy values. In addition, a current limitation of Cell-HOTA is that it only accepts flexible divisions that are one frame early or late. This could be modified to allow early or late divisions that occur two or more frames away from the ground truth. In summary, we introduce Cell-TRACTR, a novel deep learning model that can segment and track cells across subsequent frames in time-lapse images. Our model uses track queries, which leverage a cell’s history, to simultaneously segment and track cells. The self-attention mechanism in the decoder plays a crucial role in eliminating duplicate predictions and differentiating between object and track queries. We also introduced a new cell tracking metric, Cell-HOTA, and demonstrated that Cell-TRACTR performs favorably compared to other state-of-the-art cell tracking algorithms and is particularly well-suited to association and division tasks. As transformers and DETR-based models continue to progress, we anticipate that Cell-TRACTR can be adapted to increase efficiency, enabling it to handle predictions using larger images and those crowded with cells. Methods Training Cell-TRACTR For the bacterial mother machine dataset, we trained Cell-TRACTR for 12 epochs with a batch size of 2. The number of object queries was set to 30. The transformer consisted of 4 encoder layers and 4 decoder layers. The target size was 256x32 pixels. Images had to be resized to fit the target size. All configurations and hyperparameters are detailed in train_moma.yaml file within the cfgs folder on the GitLab repository. A V100 GPU was utilized. For the DeepCell dataset, we trained Cell-TRACTR for 24 epochs with a batch size of 1. The number of object queries was set to 400. The transformer consisted of 1 encoder layer and 3 decoder layers. We reduced the number of encoder and decoder layers relative to the bacterial case to minimize the computational cost. The target size was 584x600 pixels. Images were not resized since they were already the correct target size. All configurations and hyperparameters are detailed in train_DynamicNuclearNet-tracking-v1_0.yaml file within the cfgs folder on the GitLab repository. We used the A100 GPU to alleviate out-of-memory issues. Next, we detail important parameters used to train Cell-TRACTR for both datasets. We used a learning rate of 0.0002 for the encoder-decoder and a learning rate of 0.00002 for the CNN backbone. Both learning rates were dropped by a factor of 10 at epoch 12 and 20 for the models trained on the bacterial mother machine and DeepCell datasets, respectively. We used AdamW ( 76 ) as the optimizer with a weight decay of 0.0001. We adopted ResNet-18 as our backbone CNN and fed the last 3 feature maps to the encoder. The first decoder layer was used for object detection and subsequent layers were used for tracking (Fig S5, Supplementary Methods). To further reduce the computational load, we adjusted some training parameters, lowering the dimension of the queries and embeddings to 144 and lowering the dimension of the feedforward layers to 512. We used 8 attention heads. We use a clip size of up to 3 frames to train the model. The model is either fed 1, 2, or 3 frames. When 1 frame is fed to the model, only object detection is performed. When 2 frames are fed to the model, object detection is performed in the first frame and tracking is performed on the subsequent frame. When 3 frames are fed to the model, object detection is performed in the first frame and tracking is performed on the two subsequent frames. We do not exclusively feed the model 3 frames, because it needs to be trained explicitly on object detection as the loss is only calculated for the final frame within the clip. For example, when 3 frames are fed to the model, backpropagation will not affect false positives predicted during object detection in the first frame. It is important that the model learns to track for at least 3 sequential frames because the model needs to track cells that have just divided. This is a unique task for DETR-based tracking models, as cells that have just divided contain different positional embeddings but the same content embeddings (Fig S6). Following the approach used in TrackFormer ( 43 ), we utilize false negatives and false positives during training to improve the robustness of the model. False negatives are incorporated to teach the model to identify cells that were previously undetected due to errors. We implemented conditions where there is a probability of 0.5 that at least one of the track queries will be dropped, with a maximum of 20% of the track queries potentially being dropped. When a track query is dropped, an object query is forced to detect the cell. False positives are also incorporated to ensure that the model can ignore erroneous predictions made in previous frames. In our implementation, there is a probability of 0.5 that at least one false positive will added, with a maximum of 20% of the total number of track queries added as false positives. False positives are generated as noisy track queries. To generate the positional embedding for the false positive, random noise is added to the bounding box of a track query with λ 1 = 0.2 and λ2 = 0.1 (Supplementary Methods). To generate the content embedding for the false positive, random noise drawn from a normal distribution with a mean of 0 and a standard deviation of 0.25 is added to the content embedding of the respective track query. False positives and false negatives were only incorporated in the final frame of the clip when the clip size was greater than 1. Compared to TrackFormer ( 43 ), we adjusted the frequency of false positives and false negatives and the generation of the positional and content embeddings for the false positives to simplify the code. Loss function The loss function is a culmination of losses, including the class label loss, bounding box loss, and segmentation mask loss. The class label loss is calculated using focal loss with class weighting factor α = 0.25 and focusing parameter γ = 2. The bounding box loss is a linear combination of the L1 loss and generalized IOU loss. The segmentation mask loss is a linear combination of the Dice Loss and binary cross entropy. To account for cell divisions, we average the loss across the two divided cells. Therefore, a cell division would have equal weight compared to a single cell. The loss functions were taken directly from DETR ( 45 ) and modified to handle cell divisions. Data augmentations To help generalize our model, we applied three data augmentation techniques during training: random Gaussian blur, random Gaussian noise, and random illumination adjustments. All three techniques were applied with a probability of 0.4 for each augmentation for a sequence of frames. These were chosen to reflect imaging conditions seen in microscopy data. These data augmentations were all implemented using the approach described in references ( 10 , 20 ). Training DeLTA For the bacterial mother machine dataset, we trained the segmentation and tracking models for 12 epochs with a batch size of 1. The target size was 256x32 pixels. The “mother machine” configuration in DeLTA was utilized. All default data augmentations were included. For the DeepCell dataset, we trained the segmentation and tracking model for 24 epochs with a batch size of 1, incorporating a patience parameter of 5 epochs. The target size was 256x256 pixels. The “2D” configuration in DeLTA was used, in addition to all default data augmentations. Additionally, we modified the code to directly train on Cell Tracking Challenge formatted datasets. Training EmbedTrack For the bacterial mother machine dataset, we trained EmbedTrack for 12 epochs with a batch size of 16. Although cropping images is the default for EmbedTrack, the small size of the images made this step unnecessary as cropping is used to circumvent out-of-memory errors. Instead, we adjusted the code to resize all images to a uniform input size of 256x32 pixels. We removed the random rotation and random flip data augmentations from EmbedTrack, as they were incompatible with a crop size that was not a square. A V100 GPU was used for training. For the DeepCell dataset, we trained EmbedTrack for 24 epochs with a batch size of 4. The batch size was reduced from the default size of 16 to prevent out-of-memory errors. The images were cropped to 256x256 pixels, which is the default crop size for EmbedTrack. Since the images in this dataset were cropped during training and inference, there was no need for additional resizing. All default data augmentations from EmbedTrack were utilized. A V100 GPU was also used for this training. Overall, we aimed to make as minimal modifications to the code as necessary for training and testing EmbedTrack. All models were trained for 12 epochs on the bacterial mother machine dataset and 24 epochs on the mammalian DeepCell dataset. Each epoch involved iterating through every image in the dataset once, ensuring equal training time for each model. We used the default configurations and training parameters when applicable. HOTA metric overview To calculate the HOTA score, we used the approach from Ref. ( 61 ), which we summarize here for completeness. We first used the Hungarian algorithm ( 77 ) to match the tracker cells with the ground truth cells that generated the highest HOTA score. DetA is measured as the Jaccard Similarity where a match is considered a true positive (TP) if IOU > α ( Equation 1 ). False positives (FP) occur when a tracker cell does not match to any ground truth cell and false negatives (FN) occur when a ground truth cell does not match to any tracker cells. A match with an IOU below the threshold α is considered a FP for the tracker cell and a FN for the ground truth cell. To instill localization accuracy into the detection accuracy, DetA α is measured at multiple thresholds of α. In other words, the detection accuracy depends upon the value of the threshold α. To calculate AssA, for all the true positives detected at each α, the number of matches for each tracker and ground truth cell are enumerated ( Equation 2 ). A false positive association (FPA) occurs when the tracker cell matches to a different ground truth cell. A false negative association (FNA) occurs when the ground truth cell matches to a different tracker cell. A(c) is the association score for a single detected true positive where c is a detected true positive. AssA α calculates the association score over all detected true positives for a given value of α ( Equation 3 ). HOTA α is calculated by taking the geometric mean of DetA α and AssA α ( Equation 4 ). This is typically measured at values of α ranging from 0.05 to 0.95 in increments of 0.05. The final HOTA score is a calculated by taking the average Cell-HOTA α score across all α values ( Equation 5 ). Cell-HOTA metric To allow Cell-HOTA to handle cell division, we added a new sub-metric called division accuracy, DivA. DivA α is measured as the Jaccard Similarity across all cell divisions at a similarity threshold α ( Equation 6 ). DivA α is calculated using the same approach as DetA α We define AssDivA α as the geometric mean of AssA α and DivA α at a similarity threshold α ( Equation 7 ). Association and division accuracy are equally weighted to emphasize the importance of division accuracy. Cell-HOTA α is the geometric mean of DetA α and AssDivA α at a similarity threshold α ( Equation 8 ). This calculation is the same as HOTA α except AssDivA α replaces AssA α . Cell-HOTA is a measured by taking the average Cell-HOTA α score across all values of α ( Equation 9 ). This is calculated using the same approach as with HOTA. ,.2,,.2/ Cell-HOTA and flexible divisions A flexible early division occurs when a tracker cell divides at time, for example, t 1 and the matched ground truth cell divides in the next frame, t 2 in this example, (Fig S7). To check for flexible early divisions at time t 1 , we iterate through all ground truth cells that divide at time t 2 and match to a tracker cell that divides time t 1 . Next, we check that the tracker and ground truth parent cells at time t 0 match and the tracker and ground truth daughter cells match at time t 2 . Similarly, a flexible late division occurs when a ground truth cell divides at time t 1 and the matched tracker cell divides in the next frame at time t 2 . To check for flexible late divisions at time t 1 , we iterate through all tracker cells at time t 1 and check if any divide at time t 2 and match to a ground truth cell that divides at time t 1 . Next, we check that the tracker and ground truth parent cells at time t 0 match and the tracker and ground truth daughter cells match at time t 2 . For both flexible early and late divisions, we check that the average IOU between the daughter cells at time t 2 is greater than the similarity threshold α. When a flexible division is detected, the DetA α score is adjusted as if the tracker cell divided in the same frame as the ground truth cell. For example, when there is a flexible early division, the IOU is calculated between two tracker cells and one ground truth cell where the two tracker cells are treated as one object (Fig S7). When a flexible late division occurs, the IOU is calculated between the two ground truth cells and one tracker cell where the two ground truth cells are treated as one. This IOU value is counted twice since there are two ground truth cells. This is done to ensure each ground truth cell has the same weight and the score remains the same whether a tracker division was predicted at the correct time or a frame early or late. As with DetA α , we adjust the AssA α score to match the ground truth for flexible divisions. When a flexible early division occurs, a match is added between the ground truth cell and parent of the tracker cells and a match is removed from the ground truth cell and one of the daughter tracker cells. When a flexible late division occurs, a match is added between the two ground truth cells and the two divided tracker cells in the subsequent frame and a match is removed between one of the ground truth cells and the tracker cell in the current frame. Code Availability We provide the code for Cell-TRACTR at https://gitlab.com/dunloplab/Cell-TRACTR . Code for Cell-HOTA is available at https://gitlab.com/dunloplab/Cell-HOTA . Datasets The bacterial mother machine dataset is hosted on Zenodo ( https://doi.org/10.5281/zenodo.11237127 ). It is formatted in the style of the Cell Tracking Challenge. The DeepCell dataset was downloaded directly from the DeepCell website ( https://datasets.deepcell.org/data ). We used their “DynamicNuclearNet Tracking” dataset. This dataset was converted into the Cell Tracking Challenge format to be compatible for training the models. Competing Interest The authors declare no competing interests. Acknowledgements We thank Jean-Baptiste Lugagne for helpful discussions. This work was supported by the National Science Foundation grants MCB-2143289 and MCB-2032357. References 1. ↵ Brandis G , Larsson J , Elf J . Antibiotic perseverance increases the risk of resistance development . Proc Natl Acad Sci U S A . 2023 Jan 10; 120 ( 2 ): e2216216120 . OpenUrl CrossRef 2. ↵ Patel AP , Tirosh I , Trombetta JJ , Shalek AK , Gillespie SM , Wakimoto H , et al. Single-cell RNA-seq highlights intratumoral heterogeneity in primary glioblastoma . Science . 2014 Jun 20; 344 ( 6190 ): 1396 – 401 . OpenUrl Abstract / FREE Full Text 3. ↵ Choudhary D , Lagage V , Foster KR , Uphoff S . Phenotypic heterogeneity in the bacterial oxidative stress response is driven by cell-cell interactions . Cell Rep . 2023 Mar ; 42 ( 3 ): 112168 . OpenUrl 4. ↵ Sampaio NMV , Blassick CM , Andreani V , Lugagne JB , Dunlop MJ . Dynamic gene expression and growth underlie cell-to-cell heterogeneity in Escherichia coli stress response . Proc Natl Acad Sci U S A . 2022 Apr 5; 119 ( 14 ): e2115032119 . OpenUrl CrossRef 5. ↵ Li Y , Jiang Y , Paxman J , O’Laughlin R , Klepin S , Zhu Y , et al. A programmable fate decision landscape underlies single-cell aging in yeast . Science . 2020 Jul 17; 369 ( 6501 ): 325 – 9 . OpenUrl Abstract / FREE Full Text 6. ↵ Zhou Z , Liu Y , Feng Y , Klepin S , Tsimring LS , Pillus L , et al. Engineering longevity—design of a synthetic gene oscillator to slow cellular aging . Science . 2023 Apr 28; 380 ( 6643 ): 376 – 81 . OpenUrl CrossRef 7. ↵ Elowitz MB , Levine AJ , Siggia ED , Swain PS . Stochastic Gene Expression in a Single Cell . Science . 2002 Aug 16; 297 ( 5584 ): 1183 – 6 . OpenUrl Abstract / FREE Full Text 8. ↵ Cutler KJ , Stringer C , Lo TW , Rappez L , Stroustrup N , Brook Peterson S , et al. Omnipose: a high-precision morphology-independent solution for bacterial cell segmentation . Nat Methods . 2022 Nov ; 19 ( 11 ): 1438 – 48 . OpenUrl CrossRef 9. ↵ Lee G , Kim S , Kim J , Yun SY. MEDIAR: Harmony of Data-Centric and Model-Centric for Multi-Modality Microscopy [Internet] . arXiv; 2022 [cited 2023 Oct 20 ]. Available from: http://arxiv.org/abs/2212.03465 10. ↵ O’Connor OM , Alnahhas RN , Lugagne JB , Dunlop MJ. DeLTA 2.0: A deep learning pipeline for quantifying single-cell spatial and temporal dynamics . Coelho LP , editor. PLOS Comput Biol . 2022 Jan 18; 18 ( 1 ): e1009797 . OpenUrl CrossRef 11. ↵ Israel U , Marks M , Dilip R , Li Q , Schwartz M , Pradhan E , et al. A Foundation Model for Cell Segmentation [Internet] . bioRxiv; 2023 [cited 2023 Dec 11 ]. p. 2023 .11.17.567630. Available from: https://www.biorxiv.org/content/10.1101/2023.11.17.567630v2 12. ↵ van Gestel J , Bareia T , Tenennbaum B , Dal Co A , Guler P , Aframian N , et al. Short-range quorum sensing controls horizontal gene transfer at micron scale in bacterial communities . Nat Commun . 2021 Apr 19; 12 ( 1 ): 2324 . OpenUrl 13. ↵ Vliet S van, Co AD , Winkler AR , Spriewald S , Stecher B , Ackermann M . Spatially Correlated Gene Expression in Bacterial Groups: The Role of Lineage History, Spatial Gradients, and Cell-Cell Interactions . Cell Syst . 2018 Apr 25; 6 ( 4 ): 496 – 507 .e6. OpenUrl 14. ↵ Magnusson KEG , Jalden J , Gilbert PM , Blau HM . Global Linking of Cell Tracks Using the Viterbi Algorithm . IEEE Trans Med Imaging . 2015 Apr ; 34 ( 4 ): 911 – 29 . OpenUrl CrossRef PubMed 15. Bise R , Yin Z , Kanade T. Reliable cell tracking by global data association. In: 2011 IEEE International Symposium on Biomedical Imaging: From Nano to Macro [Internet] . 2011 [cited 2024 Apr 4 ]. p. 1004 – 10 . Available from: https://ieeexplore.ieee.org/document/5872571 16. Dzyubachyk O , van Cappellen WA , Essers J , Niessen WJ , Meijering E . Advanced Level-Set-Based Cell Tracking in Time-Lapse Fluorescence Microscopy . IEEE Trans Med Imaging . 2010 Mar ; 29 ( 3 ): 852 – 67 . OpenUrl CrossRef PubMed Web of Science 17. ↵ Yang X , Li H , Zhou X . Nuclei Segmentation Using Marker-Controlled Watershed, Tracking Using Mean-Shift, and Kalman Filter in Time-Lapse Microscopy . IEEE Trans Circuits Syst Regul Pap . 2006 Nov ; 53 ( 11 ): 2405 – 14 . OpenUrl 18. ↵ He T , Mao H , Guo J , Yi Z . Cell tracking using deep neural networks with multi-task learning . Image Vis Comput . 2017 Apr 1; 60 : 142 – 53 . OpenUrl 19. ↵ Meacock OJ , Durham WM . Tracking bacteria at high density with FAST, the Feature-Assisted Segmenter/Tracker . PLOS Comput Biol . 2023 Oct 9; 19 ( 10 ): e1011524 . OpenUrl 20. ↵ Lugagne JB , Lin H , Dunlop MJ. DeLTA: Automated cell segmentation, tracking, and lineage reconstruction using deep learning . Asthagiri AR , editor. PLOS Comput Biol . 2020 Apr 13; 16 ( 4 ): e1007673 . OpenUrl 21. Hayashida J , Bise R. Cell tracking with deep learning for cell detection and motion estimation in low-frame-rate: 22nd International Conference on Medical Image Computing and Computer-Assisted Intervention, MICCAI 2019 . Shen D , Yap PT , Liu T , Peters TM , Khan A , Staib LH , et al. , editors. Med Image Comput Comput Assist Interv – MICCAI 2019 - 22nd Int Conf Proc . 2019; 397 – 405 . 22. ↵ Schwartz MS , Moen E , Miller G , Dougherty T , Borba E , Ding R , et al. Caliban: Accurate cell tracking and lineage construction in live-cell imaging experiments with deep learning [Internet] . bioRxiv; 2023 [cited 2024 Jan 22 ]. p. 803205 . Available from: https://www.biorxiv.org/content/10.1101/803205v4 23. ↵ Ollion J , Maliet M , Giuglaris C , Vacher E , Deforet M. DistNet2D: Leveraging long-range temporal information for efficient segmentation and tracking [Internet] . arXiv; 2023 [cited 2023 Nov 5 ]. Available from: http://arxiv.org/abs/2310.19641 24. ↵ Loffler K , Mikut R . EmbedTrack—Simultaneous Cell Segmentation and Tracking Through Learning Offsets and Clustering Bandwidths . IEEE Access . 2022 ; 10 : 77147 – 57 . OpenUrl 25. ↵ Payer C , Štern D , Feiner M , Bischof H , Urschler M . Segmenting and tracking cell instances with cosine embeddings and recurrent hourglass networks . Med Image Anal . 2019 Oct ; 57 : 106 – 19 . OpenUrl CrossRef 26. ↵ Hayashida J , Nishimura K , Bise R. MPM: Joint Representation of Motion and Position Map for Cell Tracking . In: 2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) [Internet]. Seattle, WA, USA : IEEE ; 2020 [cited 2024 Jan 22 ]. p. 3822 – 31 . Available from: https://ieeexplore.ieee.org/document/9156603/ 27. ↵ Chen Y , Song Y , Zhang C , Zhang F , O’Donnell L , Chrzanowski W , et al. Celltrack R-CNN: A Novel End-To-End Deep Neural Network For Cell Segmentation And Tracking In Microscopy Images . In: 2021 IEEE 18th International Symposium on Biomedical Imaging (ISBI) [Internet] . Nice, France : IEEE ; 2021 [cited 2023 Feb 19 ]. p. 779 – 82 . Available from: https://ieeexplore.ieee.org/document/9434057/ 28. ↵ Hayashida J , Nishimura K , Bise R. Consistent Cell Tracking in Multi-frames with Spatio-Temporal Context by Object-Level Warping Loss . In: 2022 IEEE/CVF Winter Conference on Applications of Computer Vision (WACV) [Internet] . Waikoloa, HI, USA : IEEE ; 2022 [cited 2024 Jan 22 ]. p. 1759 – 68 . Available from: https://ieeexplore.ieee.org/document/9707080/ 29. ↵ Ollion J , Ollion C. DistNet: Deep Tracking by Displacement Regression: Application to Bacteria Growing in the Mother Machine . In: Martel AL , Abolmaesumi P , Stoyanov D , Mateus D , Zuluaga MA , Zhou SK , et al. , editors. Medical Image Computing and Computer Assisted Intervention – MICCAI 2020 [Internet] . Cham : Springer International Publishing ; 2020 [cited 2022 Apr 13 ]. p. 215 – 25 . (Lecture Notes in Computer Science; vol. 12265). Available from: https://link.springer.com/10.1007/978-3-030-59722-1_21 30. ↵ Ronneberger O , Fischer P , Brox T. U-Net: Convolutional Networks for Biomedical Image Segmentation . ArXiv150504597 Cs [Internet]. 2015 May 18 [cited 2022 Apr 13 ]; Available from: http://arxiv.org/abs/1505.04597 31. ↵ Stringer C , Wang T , Michaelos M , Pachitariu M . Cellpose: a generalist algorithm for cellular segmentation . Nat Methods . 2021 Jan ; 18 ( 1 ): 100 – 6 . OpenUrl CrossRef PubMed 32. Cutler KJ , Stringer C , Wiggins PA , Mougous JD . Omnipose: a high-precision morphology-independent solution for bacterial cell segmentation [Internet] . Microbiology ; 2021 Nov [cited 2022 Apr 13 ]. Available from: http://biorxiv.org/lookup/doi/10.1101/2021.11.03.467199 33. ↵ Panigrahi S , Murat D , Le Gall A , Martineau E , Goldlust K , Fiche JB , et al. Misic, a general deep learning-based method for the high-throughput cell segmentation of complex bacterial communities . eLife . 2021 Sep 9; 10 : e65151 . OpenUrl 34. ↵ Ben-Haim T , Raviv TR. Graph Neural Network for Cell Tracking in Microscopy Videos [Internet] . arXiv; 2022 [cited 2024 Feb 29 ]. Available from: http://arxiv.org/abs/2202.04731 35. ↵ Zhao K , Niyogisubizo J , Xiao L , Pan Y , Wei D , Rosiyadi D , et al. A Novel Deep Learning Approach Featuring Graph-Based Algorithm for Cell Segmentation and Tracking. In: 2023 IEEE International Conference on Bioinformatics and Biomedicine (BIBM) [Internet] . Istanbul , Turkiye : IEEE ; 2023 [cited 2024 Jan 23 ]. p. 1752 – 7 . Available from: https://ieeexplore.ieee.org/document/10385935/ 36. ↵ Wang J , Su X , Zhao L , Zhang J . Deep Reinforcement Learning for Data Association in Cell Tracking . Front Bioeng Biotechnol . 2020 Apr 9; 8 : 298 . 37. ↵ Vaswani A , Shazeer N , Parmar N , Uszkoreit J , Jones L , Gomez AN , et al. Attention is All you Need . : 11 . 38. ↵ Dosovitskiy A , Beyer L , Kolesnikov A , Weissenborn D , Zhai X , Unterthiner T , et al. An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. ArXiv201011929 Cs [Internet] . 2021 Jun 3 [cited 2022 Apr 13 ]; Available from: http://arxiv.org/abs/2010.11929 39. ↵ Prangemeier T , Reich C , Koeppl H. Attention-Based Transformers for Instance Segmentation of Cells in Microstructures . 2020 IEEE Int Conf Bioinforma Biomed BIBM . 2020 Dec 16; 700 – 7 . 40. ↵ Gallusser B , Weigert M. Trackastra: Transformer-based cell tracking for live-cell microscopy [Internet] . arXiv; 2024 [cited 2024 May 30 ]. Available from: http://arxiv.org/abs/2405.15700 41. ↵ Yu E , Wang T , Li Z , Zhang Y , Zhang X , Tao W. MOTRv3: Release-Fetch Supervision for End-to-End Multi-Object Tracking [Internet] . arXiv; 2023 [cited 2023 Jul 17 ]. Available from: http://arxiv.org/abs/2305.14298 42. ↵ Sun P , Cao J , Jiang Y , Yuan Z , Bai S , Kitani K , et al. DanceTrack: Multi-Object Tracking in Uniform Appearance and Diverse Motion [Internet] . arXiv; 2022 [cited 2023 Jan 16 ]. Available from: http://arxiv.org/abs/2111.14690 43. ↵ Meinhardt T , Kirillov A , Leal-Taixe L , Feichtenhofer C. TrackFormer: Multi-Object Tracking with Transformers [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2101.02702 44. ↵ Zeng F , Dong B , Zhang Y , Wang T , Zhang X , Wei Y. MOTR : End-to-End Multiple-Object Tracking with Transformer . 2021 May 7; 17 . 45. ↵ Carion N , Massa F , Synnaeve G , Usunier N , Kirillov A , Zagoruyko S. End-to-End Object Detection with Transformers [Internet] . arXiv; 2020 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2005.12872 46. ↵ Zhu X , Su W , Lu L , Li B , Wang X , Dai J. Deformable DETR: Deformable Transformers for End-to-End Object Detection [Internet] . arXiv; 2021 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2010.04159 47. ↵ Liu S , Li F , Zhang H , Yang X , Qi X , Su H , et al. DAB-DETR: Dynamic Anchor Boxes are Better Queries for DETR [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2201.12329 48. ↵ Li F , Zhang H , Liu S , Guo J , Ni LM , Zhang L. DN-DETR: Accelerate DETR Training by Introducing Query DeNoising [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2203.01305 49. ↵ Zhang H , Li F , Liu S , Zhang L , Su H , Zhu J , et al. DINO: DETR with Improved DeNoising Anchor Boxes for End-to-End Object Detection [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2203.03605 50. ↵ Zhu X , Su W , Lu L , Li B , Wang X , Dai J. Deformable DETR: Deformable Transformers for End-to-End Object Detection . ArXiv201004159 Cs [Internet]. 2021 Mar 17 [cited 2022 Apr 13 ]; Available from: http://arxiv.org/abs/2010.04159 51. ↵ Yao Z , Ai J , Li B , Zhang C. Efficient DETR: Improving End-to-End Object Detector with Dense Prior [Internet] . arXiv; 2021 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2104.01318 52. ↵ Li F , Zhang H , xu H , Liu S , Zhang L , Ni LM , et al. Mask DINO: Towards A Unified Transformer-based Framework for Object Detection and Segmentation [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2206.02777 53. ↵ Zong Z , Song G , Liu Y. DETRs with Collaborative Hybrid Assignments Training [Internet] . arXiv; 2023 [cited 2023 Jul 10 ]. Available from: http://arxiv.org/abs/2211.12860 54. ↵ Zhang Y , Wang T , Zhang X. MOTRv2: Bootstrapping End-to-End Multi-Object Tracking by Pretrained Object Detectors [Internet] . arXiv; 2022 [cited 2022 Nov 27 ]. Available from: http://arxiv.org/abs/2211.09791 55. Gao R , Wang L. MeMOTR: Long-Term Memory-Augmented Transformer for Multi-Object Tracking [Internet] . arXiv; 2023 [cited 2023 Aug 7 ]. Available from: http://arxiv.org/abs/2307.15700 56. Yan F , Luo W , Zhong Y , Gan Y , Ma L. Bridging the Gap Between End-to-end and Non-End-to-end Multi-Object Tracking [Internet] . arXiv; 2023 [cited 2023 Jul 10 ]. Available from: http://arxiv.org/abs/2305.12724 57. Liu Y , Wu J , Fu Y. Collaborative Tracking Learning for Frame-Rate-Insensitive Multi-Object Tracking [Internet] . arXiv; 2023 [cited 2023 Aug 16 ]. Available from: http://arxiv.org/abs/2308.05911 58. ↵ Gao R , Zhang Y , Wang L. Multiple Object Tracking as ID Prediction [Internet] . arXiv; 2024 [cited 2024 Mar 31 ]. Available from: http://arxiv.org/abs/2403.16848 59. ↵ Maška M , Ulman V , Delgado-Rodriguez P , Gómez-de-Mariscal E , Nečasová T , Guerrero Peña FA , et al. The Cell Tracking Challenge: 10 years of objective benchmarking . Nat Methods . 2023 May 18; 1 – 11 . 60. ↵ Chen Y , Huo Y. arXiv.org . 2020 [cited 2023 Jun 8 ]. Limitation of Acyclic Oriented Graphs Matching as Cell Tracking Accuracy Measure when Evaluating Mitosis . Available from: https://arxiv.org/abs/2012.12084v1 61. ↵ Luiten J , Osep A , Dendorfer P , Torr P , Geiger A , Leal-Taixe L , et al. HOTA: A Higher Order Metric for Evaluating Multi-Object Tracking . Int J Comput Vis . 2021 Feb ; 129 ( 2 ): 548 – 78 . OpenUrl 62. ↵ Wang P , Robert L , Pelletier J , Dang WL , Taddei F , Wright A , et al. Robust growth of Escherichia coli . Curr Biol CB . 2010 Jun 22; 20 ( 12 ): 1099 – 103 . OpenUrl 63. ↵ Zhang Y , Sun P , Jiang Y , Yu D , Weng F , Yuan Z , et al. ByteTrack: Multi-Object Tracking by Associating Every Detection Box [Internet] . arXiv; 2022 [cited 2022 Nov 14 ]. Available from: http://arxiv.org/abs/2110.06864 64. Wang YH , Hsieh JW , Chen PY , Chang MC , So HH , Li X. SMILEtrack: SiMIlarity LEarning for Occlusion-Aware Multiple Object Tracking [Internet] . arXiv; 2023 [cited 2023 Oct 20 ]. Available from: http://arxiv.org/abs/2211.08824 65. ↵ Bewley A , Ge Z , Ott L , Ramos F , Upcroft B . Simple Online and Realtime Tracking . In: 2016 IEEE International Conference on Image Processing (ICIP) [Internet] . 2016 [cited 2022 Nov 14 ]. p. 3464 – 8 . Available from: http://arxiv.org/abs/1602.00763 66. ↵ He K , Zhang X , Ren S , Sun J. Deep Residual Learning for Image Recognition . In: 2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR) [Internet] . Las Vegas, NV, USA : IEEE ; 2016 [cited 2022 Apr 13 ]. p. 770 – 8 . Available from: http://ieeexplore.ieee.org/document/7780459/ 67. ↵ Bernardin K , Stiefelhagen R . Evaluating multiple object tracking performance: the clear mot metrics . EURASIP Journal on Image and Video Processing . 2008 ;2008: 1 – 10 . 68. ↵ Ristani E , Solera F , Zou RS , Cucchiara R , Tomasi C. Performance Measures and a Data Set for Multi-Target, Multi-Camera Tracking [Internet] . arXiv; 2016 [cited 2023 Oct 2 ]. Available from: http://arxiv.org/abs/1609.01775 69. ↵ Dendorfer P , Rezatofighi H , Milan A , Shi J , Cremers D , Reid I , et al. CVPR19 Tracking and Detection Challenge: How crowded can it get? [Internet] . arXiv; 2019 [cited 2024 Feb 6 ]. Available from: http://arxiv.org/abs/1906.04567 70. ↵ Liang Z , Yuan Y. Mask Frozen-DETR: High Quality Instance Segmentation with One GPU [Internet] . arXiv; 2023 [cited 2023 Aug 9 ]. Available from: http://arxiv.org/abs/2308.03747 71. ↵ Lv W , Zhao Y , Xu S , Wei J , Wang G , Cui C , et al. DETRs Beat YOLOs on Real-time Object Detection [Internet]. arXiv; 2023 [cited 2024 Jan 19 ]. Available from: http://arxiv.org/abs/2304.08069 72. ↵ Lin J , Mao X , Chen Y , Xu L , He Y , Xue H. D^2ETR: Decoder-Only DETR with Computationally Efficient Cross-Scale Attention [Internet] . arXiv; 2022 [cited 2024 Jan 20 ]. Available from: http://arxiv.org/abs/2203.00860 73. ↵ Dubey S , Olimov F , Rafique MA , Jeon M. Improving Small Objects Detection using Transformer . 74. ↵ Huang J , Wang H. Small Object Detection by DETR via Information Augmentation and Adaptive Feature Fusion [Internet] . arXiv; 2024 [cited 2024 Jan 19 ]. Available from: http://arxiv.org/abs/2401.08017 75. ↵ Xu Y , Lyu D , Li Z , Wang Z , Chen Y , Wang G , et al. DEFA: Efficient Deformable Attention Acceleration via Pruning-Assisted Grid-Sampling and Multi-Scale Parallel Processing [Internet] . arXiv; 2024 [cited 2024 Mar 27 ]. Available from: http://arxiv.org/abs/2403.10913 76. ↵ Loshchilov I , Hutter F. Decoupled Weight Decay Regularization. ArXiv171105101 Cs Math [Internet] . 2019 Jan 4 [cited 2022 Apr 13 ]; Available from: http://arxiv.org/abs/1711.05101 77. ↵ Kuhn HW . The Hungarian method for the assignment problem . Nav Res Logist Q . 1955 Mar ; 2 ( 1– 2 ): 83 – 97 . OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted July 16, 2024. Download PDF Supplementary Material Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells Owen M. O’Connor , Mary J. Dunlop bioRxiv 2024.07.11.603075; doi: https://doi.org/10.1101/2024.07.11.603075 Share This Article: Copy Citation Tools Cell-TRACTR: A transformer-based model for end-to-end segmentation and tracking of cells Owen M. O’Connor , Mary J. Dunlop bioRxiv 2024.07.11.603075; doi: https://doi.org/10.1101/2024.07.11.603075 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Systems Biology Subject Areas All Articles Animal Behavior and Cognition (7646) Biochemistry (17728) Bioengineering (13917) Bioinformatics (42038) Biophysics (21489) Cancer Biology (18637) Cell Biology (25554) Clinical Trials (138) Developmental Biology (13403) Ecology (19941) Epidemiology (2067) Evolutionary Biology (24368) Genetics (15624) Genomics (22547) Immunology (17764) Microbiology (40475) Molecular Biology (17208) Neuroscience (88756) Paleontology (667) Pathology (2842) Pharmacology and Toxicology (4834) Physiology (7659) Plant Biology (15175) Scientific Communication and Education (2047) Synthetic Biology (4304) Systems Biology (9835) Zoology (2272)