A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care

A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care View ORCID Profile Jamie Kurtzig , View ORCID Profile Johannes O. Ferstad , View ORCID Profile Paul Dupenloup , View ORCID Profile Dessi P. Zaharieva , View ORCID Profile David M. Maahs , View ORCID Profile Priya Prahalad , View ORCID Profile Ramesh Johari , Franziska K. Bishop , View ORCID Profile Ananta Addala , View ORCID Profile David Scheinker doi: https://doi.org/10.1101/2025.02.03.25321621 Jamie Kurtzig 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Jamie Kurtzig Johannes O. Ferstad 2 Department of Management Science and Engineering, Stanford University School of Engineering , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Johannes O. Ferstad Paul Dupenloup 2 Department of Management Science and Engineering, Stanford University School of Engineering , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Paul Dupenloup Dessi P. Zaharieva 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Dessi P. Zaharieva David M. Maahs 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA 3 Stanford Diabetes Research Center, Stanford University , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for David M. Maahs Priya Prahalad 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA 3 Stanford Diabetes Research Center, Stanford University , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Priya Prahalad Ramesh Johari 2 Department of Management Science and Engineering, Stanford University School of Engineering , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ramesh Johari Franziska K. Bishop 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site Ananta Addala 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA 3 Stanford Diabetes Research Center, Stanford University , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Ananta Addala David Scheinker 1 Division of Endocrinology, Department of Pediatrics, Stanford University School of Medicine , California, USA 2 Department of Management Science and Engineering, Stanford University School of Engineering , California, USA 3 Stanford Diabetes Research Center, Stanford University , California, USA 4 Clinical Excellence Research Center, Stanford University , California, USA Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for David Scheinker For correspondence: dscheink{at}stanford.edu Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Introduction Clinics continue to adopt remote patient monitoring for type 1 diabetes (T1D) and care models shaped by algorithmic CGM data analysis. No clinic-facing quantitative framework currently exists to track the impact of such algorithm-directed care on patient outcomes and clinical workload. The Teamwork, Targets, Technology, and Tight Control (4T) Study provides precision, whole-population care enabled by algorithms that use continuous glucose monitoring (CGM) data to direct clinician attention to patients with deteriorating glucose management. Methods We used data from the 4T Pilot (n=133) and 4T Study 1 (n=135), in which algorithms use CGM data to identify youth with T1D meeting criteria for clinical review and potential clinician contact. Through iterative data analysis and interviews with diabetes educators and clinicians, we identified metrics for reviewing and revising clinical workloads, glucose management, and timeliness of care. For each metric, we developed an interactive dashboard to provide clinical and administrative leaders with an overview of the program. Results The metrics to track clinical workload were the total number of youths: (1) in the program, (2) in each study, and (3) cared for by each clinician. The metrics to track glucose management were the number of youths meeting each criterion for review: (4) in total, (5) for each clinician, and (6) for each study. The metric to track timeliness of care was (7) the number of days since meeting criteria for clinical review. When presented at weekly program leadership meetings, the metrics facilitated data-driven decision making about clinical and operational components of the program. Conclusion We propose a novel quantitative framework for diabetes care teams to supervise and enhance algorithm-directed whole-population T1D care. As the role of algorithms grows in directing clinical effort and prioritizing patients for care, this framework may help clinics track clinical workload, patient outcomes, and the timeliness of care. Introduction Remote patient monitoring (RPM) programs for type 1 diabetes (T1D) care often use data provided by continuous glucose monitoring (CGM) systems. 1 – 4 RPM for T1D has been shown to improve HbA1c, quality of life, and glucose metrics in a variety of healthcare settings. 5 – 7 Numerous centers report the use of RPM for T1D care, and many include some level of algorithmic support in identifying patients for prioritization and clinical review or contact. 8 – 12 In the Teamwork, Targets, Technology, and Tight Control (4T) Study 1 cohort, care directed based on the analysis of CGM data equitably reduced median HbA1c to 6.58% at 12 months after diagnosis. 12 The growing use of CGM is driving the demand for algorithms to manage data and inform care delivery, increasing the complexity of clinic operations. 13 To our knowledge, no clinician-facing quantitative framework is available to track how algorithm-directed care impacts clinical workload, patient glucose management, and timeliness of care. Such quantitative frameworks may be helpful for clinics that: remotely access patient data; provide RPM-based care; and employ algorithms to direct care delivery by, for example, identifying or prioritizing patients requiring care. Although our work focuses on RPM in a pediatric T1D population, these concepts may translate to other chronic health conditions in which algorithms support and guide clinical care. 14 – 16 The Stanford 4T Study whole-population RPM tool, Timely Interventions for Diabetes Excellence (TIDE), uses CGM data to prioritize patients with T1D for personalized review and contact by the Certified Diabetes Care and Education Specialist (CDCES). 13 , 17 The 4T Study and the use of TIDE have been associated with significant, equitable improvements in glycemic outcomes, as well as reduced CDCES workload and burden. 17 – 22 The aim of this work is to develop a set of clinical and operational metrics for an organization to monitor and improve population-level, data-driven T1D care. Methods Setting The 4T Study aims to improve care for newly diagnosed youth with T1D in Stanford’s Pediatric Endocrinology clinic. The study was approved by Stanford’s Institutional Review Board (clinicaltrials.gov: NCT03968055 , NCT04336969 ). Participants ≥ 18 years of age or legal guardians of minors provided informed consent, while youth ≥ 7 years of age provided assent prior to study initiation. Participants in the 4T Study were between the ages of 6 months and 21 years and started on CGM (Dexcom G6; Dexcom Inc., San Diego, CA) within the first month of T1D diagnosis. A summary of the program is provided below, and details of the program have been published. 12 The 4T Study team performed asynchronous remote patient monitoring of CGM data from participants using an algorithm-enabled care platform, Timely Interventions for Diabetes Excellence (TIDE). The 4T Study includes the Pilot 4T study and 4T Study 1, as well as long-term follow-ups for each study. The long-term follow-up programs assess the ongoing impact of the Pilot 4T and Study 1 over an extended period. The Pilot 4T Study (n=135) enrolled participants from July 2018 to June 2020. 10 , 18 Participants diagnosed between March 2019 and January 2020 were initially offered non-TIDE RPM (e.g. Dexcom Clarity), which provided individual T1D insights rather than population-wide overviews like TIDE. These participants transitioned to TIDE when it was launched in January 2020. Participants diagnosed during and after January 2020 were enrolled into TIDE for RPM. 4T Study 1 (n=133) enrolled participants from 2020 to 2022 and focused on initiating CGM use within the first 30 days after T1D diagnosis. In both the Pilot 4T and Study 1, participant data were initially reviewed weekly for the first year after T1D diagnosis and then monthly thereafter. 23 After the conclusion of each study, participants had the option to participate in an ongoing long-term follow-up program with monthly data reviews conducted by a CDCES. RPM Platform Participants uploaded their CGM data via a personal smart device or study-provided iPod Touch (Apple Inc., Cupertino, CA) to Dexcom Clarity. TIDE automatically pulled CGM data from the 4T Study’s Dexcom Clarity clinic account using a Python script, analyzed the data, and displayed results in an interactive data visualization dashboard called Tableau. Guided by CGM clinical consensus metrics, the TIDE algorithm ranked participants based on the urgency of data review and contact by a CDCES. 20 , 20 , 23 , 24 Each week, the TIDE platform tracked whether participants met the following clinical categories: spending less than 65% of CGM time in range (TIR; 70-180 mg/dL), over 1% time below range (TBR) level 2 (below 55 mg/dL), a drop in TIR over 15% points from the previous week, over 4% TBR level 1 (below 70 mg/dL), over 50% missing CGM data, or meeting clinical targets. 25 Meeting targets refers to all participants that do not meet any of the other criteria (i.e., they have > 65% TIR, < 1% TBR level 2, < 15% drop in TIR, < 4% TBR level 1, and < 50% missing CGM data). Some metrics were updated between the Pilot Study and Study 1; for example, the TIR target was changed from 65% to 70%, changing the criteria flagged. The CDCES team members used TIDE to view a list of participants, ranked by their clinical risk likelihood as defined by the CGM metrics listed above. CDCES team members then took appropriate follow-up steps if needed, such as sending secure messages with care recommendations to participants through the Electronic Health Record (EHR). Metrics and Key Performance Indicators (KPIs) A metric is a quantitative measure of an aspect of a care process or the patient population. A key performance indicator (KPI) combines metrics into a high-level measure or visualization that reflects strategic goals and provides insights into the overall success and performance of an organization. The KPIs presented in this paper include clinical workload, glucose management, and timeliness of care. Developing a Quantitative Framework for Clinic Management In 2022, we began developing a standardized approach to monitor the clinical workload, patient glucose management, and timeliness of care associated with the use of TIDE. Numerous ad-hoc metrics were developed and measured based on the experiences of the 4T Study team, CDCES team, and administrative team. 20 The care team evaluated each proposed metric. We discarded metrics that were not clearly interpretable, did not translate into a useful KPI, or did not inform a clinical or operational decision. The next stage was an analytical approach to refine the metrics based on an analysis of the data generated, structured interviews with the CDCES team, and discussions with the research and clinical staff. In interviews, the stated goals of the process were to establish a more robust and comprehensive set of metrics for monitoring clinical workload, patient glucose management, and timeliness of care in the presence of algorithm-directed care. The above process of developing, evaluating, and soliciting feedback was used iteratively to develop metrics and KPIs based on these metrics. If a metric was interpretable and had the potential to drive a specific decision, we developed multiple visual representations. We shared these visualizations with the CDCES team and the clinicians in the 4T Study and asked them to identify specific decisions that could be informed by the metric. After receiving feedback, we refined the definition and visual presentation of each metric and eliminated from further consideration metrics that did not inform specific decisions. We asked the care team if each metric was appropriate and how it would be interpreted. Along with the definition of each finalized metric, we provided the KPIs that the metric is used to generate, and illustrative examples of the decisions it supports. Visualization of Metrics We developed an instantiation of this framework with clinic-specific metrics deployed in an interactive dashboard engineered to streamline the monitoring and visualization of the entire program. We note that participants were enrolled in and departed from the 4T Study continuously; thus, the number of participants shown at any time in the visualizations is less than the total study population. Practical Use of the Dashboard for the Visualization of Metrics At the weekly 4T study team meeting with clinical and administrative 4T team members, the dashboard was presented, and the metrics were discussed to facilitate an overview of the program. Feedback from the 4T team on potential improvements to the metrics and associated visualizations were incorporated. All visualizations developed in Tableau were double-checked with manual calculations. Obtaining Feedback We evaluated the perceived utility of this framework through interviews with the care team and discussions at the weekly 4T team meeting. The interviews were designed to collect qualitative insights into the clinical team’s firsthand experiences, perceptions, and suggestions regarding the framework’s implementation and impact. We ensured that a diverse range of perspectives were represented, including CDCES team members, clinical research coordinators, clinicians, engineers, and research staff. Results Of the 268 participants in the 4T Pilot and Study 1, a total of 222 were eligible for RPM and were included in the KPI framework ( Table 1 ). In the Pilot Study, 46 patients were ineligible, as TIDE had not yet been developed. 10 The metrics were calculated from data continuously pulled from servers on each participant’s data completeness, the study they were enrolled in, the most recent week they were shown in the TIDE dashboard, their CGM data, and the CDCES responsible for reviewing their data. View this table: View inline View popup Table 1. Participant demographics for the 4T Pilot and 4T Study 1 To monitor clinical workload, the following metrics were identified: (1) the number of youths in the 4T program, (2) in each study, and (3) cared for by each CDCES. For tracking participant glucose management, the metrics include the number of youths meeting each criterion for clinical review, both (4) overall and for each (5) CDCES and (6) study. Initially, the metric for tracking the timeliness of care was the number of days since each participant’s last day meeting criteria for clinical review. This was later revised to (7) include only participants who have not been seen in over 20 days and rank in the top 25% of patients with the longest time since meeting criteria. Feedback from all care team members highlighted these seven metrics as essential for monitoring clinical workload, patient glucose management, and timeliness of care ( Table 2 ). The detailed definitions of each metric are provided in the appendix. Each metric is associated with a KPI displayed in an interactive figure with various filters and categorizations ( Table 2 ). View this table: View inline View popup Table 2. Included and excluded metrics, the corresponding key performance indicators (KPIs), and the decisions the metrics drive To visualize each metric related to clinical workload, we used different colors to represent the number of patients: requiring review, meeting targets, and missing >50% CGM data. For each of those three criteria, this was reported as the number of distinct patient IDs. In the visualization of metrics related to patient outcomes, we used different colors to represent the number of patients in each clinical category. The clinical categories are less than 65% TIR, over 1% TBR level 2, over 4% TBR level 1, over a 15% drop in TIR, and less than 50% CGM wear time. For each clinical category, this is defined as the number of distinct patient IDs of participants flagged. For the visualization of the metric related to timeliness of care, we displayed participant information only if more than 20 days had passed since the participant was last shown in the TIDE dashboard, and the participant was in the highest quartile for delayed care. The threshold of 20 days was selected based on a retrospective analysis of historical TIDE data, which revealed that the 75th percentile of time between participant dashboard appearances was 20 days. Throughout this process, three metric categories were discarded because they were difficult to interpret or did not support a clinical or operational decision. The first were measures of the number of CDCES messages/contacts through the EHR, which included metrics such as overall message count, contact count, TIR by message count, CGM time worn by message count, patients contacted versus suggested for contact, and patient response rate to clinician messages ( Table 2 ). Due to limited CDCES time, it was decided that these metrics would not drive clinical decisions. Second were measures of how diabetes duration may impact the clinical categories and CGM metrics, as well as the average diabetes duration for each of the CDCES’ patients ( Table 2 ). These were ultimately excluded since they would not drive day-to-day clinical or operational decisions and would be more appropriate for investigation in retrospective analyses. Third were measures of participants’ average glucose ( Table 2 ). These were excluded since TIR is highly correlated and considered a more comprehensive measure. 25 , 26 For all metrics and visualizations presented, gaps or valleys in the data occur when, primarily due to the cadence of reviews, fewer CDCESs used TIDE during that period. Metrics and Visualizations for Clinical Workload The first metric was the total number of participants shown in the TIDE program. By monitoring the number of participants shown in TIDE, this metric allowed for the tracking of population statistics over time. The visualization of the metric selected by the CDCES team included the total number of participants in the program over time, segmented by those that did and did not require review ( Figure 1 ). The periodicity in the figure may be explained by the unequal number of participants assigned to each week ( Figure 1 ). The repetitive patterns in the data are due to the cadence of CDCES reviews. For example, 4T Pilot initially reviewed participants weekly but switched to monthly after 1 year of diagnosis, which contributes to the large decrease in participants seen around week 40 of 2022. One decision potentially driven by this metric was investigating recruitment efforts or participant withdrawal rates if the metric were to be unexpectedly low or high. This metric was also used to investigate the stringency of the criteria for participant review if the numbers of participants being reviewed were too high or too low. Download figure Open in new tab Figure 1. Weekly participant counts in the Timely Interventions for Diabetes Excellence (TIDE) dashboard, categorized by participants requiring review (blue), meeting targets (green), and with over 50% missing data (red). The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. The second metric was the number of participants shown in TIDE categorized by each CDCES in the program. CDCES names are numbered to protect anonymity. This metric revealed significant differences in the number of participants assigned to each CDCES, as well as stability in the number of participants reviewed by each CDCES over time ( Figure 2 ). Differences in the number of patients assigned to each CDCES can be attributed to the level of their patients’ complexity, their patients’ primary languages, and each CDCES’ time available to review participant data. This metric allowed for the monitoring of CDCES workload and identifying which CDCES team members may experience heavier participant workloads and require additional support, enabling workload balancing. For example, this metric captured the redistribution of participants from CDCES 1 to CDCES 2 in Week 12 of 2023 ( Figure 2 ). Download figure Open in new tab Figure 2. Weekly participant counts per CDCES in the TIDE dashboard, categorized by participants requiring review (blue), meeting targets (green), and with over 50% missing data (red). The dashed lines mark the transitions of participants between CDCES team members. The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. Similarly, this metric showed that CDCES 5 stopped reviewing data in Week 11 of 2024, and CDCES 4 took on the review of their participants ( Figure 2 ). The third metric was the weekly number of participants shown in TIDE per study. This metric may help clinics with several studies view the number of participants in the RPM program for each study. Alternatively, this metric may be used for tracking the number of patients within various populations, such as the number of patients that are pediatric versus adult, within 1 year of T1D diagnosis, at potentially high risk for hypoglycemia, pregnant, non-English speaking, recently hospitalized, diagnosed with certain medical conditions or family histories, or using specific tools to manage their health. The major drop in weekly participants in 2022 corresponded to the switch to the 4T Pilot long-term follow up program, which reviewed participants monthly instead of weekly ( Figure 3 ). The sharp decline in week 40 of 2022 occurred because all patients who had not yet reached a one-year duration in the 4T Study were moved to the long-term follow-up phase following the conclusion of 4T Pilot. Download figure Open in new tab Figure 3. Weekly participant counts per study in the TIDE dashboard, categorized by participants requiring review (blue), meeting targets (green), and with over 50% missing data (red). The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. Metrics and Visualizations for Patient Outcomes The fourth metric was the number of participants shown in TIDE per clinical category. This metric identified the level of variability in glucose management for each participant over time ( Figure 4 ). This metric may help to monitor participants’ data and identify areas of improvement or concern, allowing CDCES team members to adjust care plans and interventions. For example, an increase in the number of participants missing glucose data may prompt CDCES team members to educate patients on how to improve connectivity with their CGM and how to upload glucose data. Download figure Open in new tab Figure 4. Weekly participant counts per clinical category in the TIDE dashboard, categorized by participants requiring review (blue), meeting targets (green), and with over 50% missing data (red). The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. The fifth metric was the number of participants shown in TIDE per clinical category and reviewing CDCES. This metric identified the similarities and differences in glucose management for the participants reviewed by each CDCES ( Figure 5 ). In our clinic, the metric revealed that all CDCES team members had similar proportions of participants in each clinical category ( Figure 5 ). This metric can be used to help track how well each CDCES’ participants were meeting clinical consensus guidelines for CGM targets. The drop-off across all clinical categories before week 49 occurred because 4T Pilot ended, and not all participants chose to continue in the long-term follow-up program. Additionally, while 4T Pilot reviewed patient data weekly, the long-term follow-up program did so monthly, resulting in fewer participants appearing in the TIDE dashboard. This decline is most apparent in the green “Meeting Targets” category, as participants not meeting targets are prioritized and flagged for review over those who are meeting them. Download figure Open in new tab Figure 5. Weekly participant counts per CDCES in the TIDE dashboard, categorized by participants meeting targets (blue), spending over 1% of CGM readings below 55 mg/dL (orange), spending over 4% CGM readings below 70 mg/dL (green), spending less than 65% of CGM time in range (TIR; 70-180 mg/dL; red), experiencing over 15% drop in TIR (purple), and missing over 50% CGM data (brown). The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. The sixth metric showed the weekly participant counts per study and clinical category. This metric may allow clinics to compare glucose management across studies or populations. The use of this metric allowed the 4T team to identify variation in the numbers of participants in different studies and how often the groups were reviewed by CDCES team members ( Figure 6 ). This provided insight into how well participants in different studies met clinical consensus glycemic metrics. This metric facilitated easy comparison of health differences between groups, the difference between the outcomes of participants seen weekly versus monthly, and the clinical benefits of different study protocols. Download figure Open in new tab Figure 6. Weekly participant counts per study in the TIDE dashboard, categorized by participants meeting targets (blue), spending over 1% of CGM readings below 55 mg/dL (orange), spending over 4% CGM readings below 70 mg/dL (green), spending less than 65% of CGM time in range (TIR; 70-180 mg/dL; red), experiencing over 15% drop in TIR (purple), and missing over 50% CGM data (brown). The grey area represents the duration of 4T Pilot, and the white area represents the ongoing duration of the 4T Pilot long-term follow-up program. Metric and Visualization for Timeliness of Clinical Reviews The seventh metric was originally the number of days since each participant’s most recent date of being displayed in TIDE. The patients presented were sorted from least to most recent display in TIDE and presented along with the participant’s patient ID, the CDCES that reviewed their data, and the study that participants were enrolled in. After review of this metric at a 4T leadership meeting, several alternative metrics and visualizations were proposed. The chosen visualization was a table comprising participants who have not been shown in over 20 days and ranked in the top 25% of patients with the longest time since being shown. These restrictions focused on the patients most likely to benefit from review and limit the number of patients to be reviewed by the CDCES team. In addition to showcasing the days since their last appearance in TIDE, the table included the participant’s patient ID, the CDCES responsible for them, and the study they were enrolled in. If a participant had not been flagged in the RPM program in the last 20 days, this figure allowed the CDCES team to message or schedule a visit with participants, making sure no participants were inadvertently ignored. Similarly, this metric allowed clinical research coordinators to remove certain participants from studies that they were no longer enrolled in. For example, based on this metric, research coordinators checked if the participants that have not been shown in the TIDE dashboard in over 50 days are still enrolled. Feedback from the Care Team Administrative, clinical, and research feedback validated the proposed framework. Metrics were retained only if identified as helpful in understanding and visualizing clinic KPIs. Discussion We developed a quantitative framework for monitoring clinical workload, patient glucose management, and timeliness of care for a whole-population T1D care model in which an algorithm analyzes CGM data to help direct care providers to patients with CGM metrics not meeting targets. Through a multi-year process of data analysis, data visualization, and feedback from the CDCES team and clinic leadership, we identified metrics that informed oversight and decision-making for the algorithm-enabled care model. This framework may be a resource for other pediatric T1D clinics seeking to optimize their RPM-based care strategies around algorithms that help direct clinician efforts. These metrics yielded valuable insights into the RPM program and may potentially provide insights into health disparities. First, despite efforts to evenly distribute CDCES workloads, individuals meeting criteria for review were coincidentally grouped together during certain review periods, which may result in less time dedicated to individuals meeting criteria for review. This finding prompted the care team to consider redistributing subsequent enrollees. Second, the number of participants shown in TIDE per clinical category and reviewing CDCES (the fifth metric) may be used as a starting point to compare the performance of CDCES team members. Meaningful comparisons will require significant additional controls and considerations, such as patient complexity, engagement, and level of access to healthcare. Third, this metric may be used as a tool to investigate health disparities and the impact of congruence on the relationship between CDCESs and their participants. With this data, we may look into the disparity in usage of diabetes technologies, incongruence in clinician and patient perceptions of diabetes technologies, as well as how interest and feasibility of diabetes technology programs for patients with public insurance changes over time. 27 Finally, this tool has the potential to identify positive outliers (CDCES team members whose patients achieve clinical targets) or to monitor the results of pilot interventions deployed by a subset of the CDCESs. The ability to track clinical categories can be valuable in making decisions related to clinical efficiency and care delivery. Guided by insights from this metric, CDCES team members could stage interventions to improve glucose management or CGM wear time, compare our clinic’s TIR to other clinics, or observe how seasonal variations impact TIR. The proposed framework is the first quantitative approach, of which we are aware, to monitor an entire RPM program’s outcomes, process metrics, and workloads. Developing metrics that drive decision making is critical, particularly in an era where algorithms can significantly influence patient care and clinical workload. In our clinic, the review of these algorithm-enabled KPIs is becoming standard practice, integral to monitoring and adjusting a care model in which algorithms reduce clinical workload. How to Implement KPIs and Metrics at a Clinic The use of metrics to design KPIs can benefit large clinics or hospital systems. A large clinic or hospital might begin developing metrics by collaborating across multiple departments, potentially including internet technology specialists, data scientists, clinical research coordinators, clinicians, and quality improvement specialists. Clinics or hospitals may protect the time of or hire dedicated staff to implement the framework and help with data analysis and visualization. We recommend that the internet technology team explores several platforms to host a dashboard for visualizing their metrics. Since large clinics may face scalability or data integration issues, it could be beneficial to partner with technology vendors or research collaborators. In a collaborative effort between Stanford’s 4T team and Tidepool, a diabetes technology non-profit, Tidepool-TIDE, a new clinic-agnostic, turnkey solution available to any clinic in the United States was developed. 28 Limitations & Future Metrics A limitation of the framework is the lack of data on the amount of time CDCES team members spend in the TIDE dashboard. Tableau, where TIDE is currently hosted, cannot track the amount of time that CDCES team members spend reviewing participant data. While the number of patients cared for by a CDCES is a useful proxy of workload, metrics that directly measure CDCES screen time would be ideal. Two areas for further improvement of the framework are the planned inclusion of participant reported outcomes, which are already being collected as part of the 4T program, 11 and metrics from physical activity trackers, such as heart rate and step count, as they become integrated into TIDE. 13 Conclusion Data from CGMs, insulin pumps, and automated insulin delivery systems is becoming increasingly important in the management of T1D. To best utilize the data, clinics increasingly turn to trusted, transparent, and user-friendly algorithms to translate patient data into insights that inform patient care. As the impact of such algorithms grows, the proposed framework may offer clinics a quantitative approach to monitor and potentially adjust the care delivery model. Data Availability All data produced in the present study are available upon reasonable request to the authors. Authorship J.K.: all (lead). J.O.F: conceptualization (equal); software (supporting). P.D: conceptualization (equal); software (supporting). D.P.Z: review and editing (supporting); methodology (supporting). D.M.M: review and editing (supporting); conceptualization (supporting); methodology (supporting). P.P: review and editing (supporting); conceptualization (supporting); methodology (supporting). R.J.: review and editing (supporting); methodology (supporting). F.K.B.: review and editing (supporting); methodology (supporting). A.A.: review and editing (supporting); methodology (supporting). D.S.: review and editing (equal); conceptualization (equal); methodology (equal). Disclosures D.M.M., D.S., P.P., and R.J. have received support from Stanford MCHRI, Stanford HAI and the NSF. D.S., R.J., D.M.M., D.P.Z. and A.A. have received funding from the Helmsley Charitable Trust. D.P.Z. has received honoraria for speaking engagements from Ascensia Diabetes, Insulet Canada, Medtronic Diabetes, and Dexcom; research support from the ISPAD-JDRF Research Fellowship; and serves on the advisory board of DexCom Inc. A.A. has received research support from the NIH, Maternal Child Health Research Institute, and grants K12DK122550 (Stanford University) and K23DK131342 from the NIDDK during the conduct of the study. D.M.M. has had research support from the NIDDK, NIH, Breakthrough T1D, and grant number P30DK116074; his institution has had research support from DexCom Inc., Medtronic, Insulet, Bigfoot Biomedical, Tandem, and Roche; and has consulted for Abbott, the Leona M. and Harry B. Helmsley Charitable Trust, Lifescan, Sanofi, Medtronic, Provention Bio, Kriya, Novo Nordisk, Eli Lilly, Insulet, Biospex, and Bayer. J.F. has received support from an NSF grant and is now employed by OpenAI. D.S. is an adviser to Carta Health. J.K. interned at Dexcom. All other authors declare that they have no competing interests. Funding NIDDK - R18DK122422 Helmsley Charitable Trust (G-2002-04251-2) ISPAD-JDRF Research Fellowship SDRC (1P30DK 11607401) LPCH Auxiliaries National Science Foundation (2205084) Stanford HAI Stanford REDCap Platform (UL1 TR003142) Acknowledgements The authors would like to acknowledge all youth who participated in the 4T Study. We would like to thank the other members of the research team including research coordinators, clinical staff, students in the Systems Utilization Research for Stanford Medicine (SURF) group, the Quantitative Sciences Unit, the T1D Working Group in Statistics, and Informatics at Stanford Medicine Children’s Health. Abbreviations T1D Type 1 diabetes CGM Continuous glucose monitor 4T Teamwork, Targets, Technology, and Tight Control RPM Remote patient monitoring TIDE Timely Interventions for Diabetes Excellence CDCES Certified Diabetes Care and Education Specialist EMR Electronic medical record KPI Key performance indicator TIR Time in range TBR Time below range References 1. ↵ Su D , Michaud TL , Estabrooks P , et al. Diabetes Management Through Remote Patient Monitoring: The Importance of Patient Activation and Engagement with the Technology . Telemed E-Health 2019 ; 25 ( 10 ): 952 – 959 ; doi: 10.1089/tmj.2018.0205 . OpenUrl CrossRef 2. Mann DM , Chen J , Chunara R , et al. COVID-19 transforms health care through telemedicine: Evidence from the field . J Am Med Inform Assoc 2020 ; 27 ( 7 ): 1132 – 1135 ; doi: 10.1093/jamia/ocaa072 . OpenUrl CrossRef PubMed 3. Laffel LM , Kanapka LG , Beck RW , et al. Effect of Continuous Glucose Monitoring on Glycemic Control in Adolescents and Young Adults With Type 1 Diabetes: A Randomized Clinical Trial . JAMA 2020 ; 323 ( 23 ): 2388 ; doi: 10.1001/jama.2020.6940 . OpenUrl CrossRef PubMed 4. ↵ Rodbard D. Continuous Glucose Monitoring: A Review of Successes, Challenges, and Opportunities . Diabetes Technol Ther 2016 ; 18 ( S2 ): S2–3 - S2-13 ; doi: 10.1089/dia.2015.0417 . OpenUrl CrossRef PubMed 5. ↵ American Diabetes Association Professional Practice Committee . 7. Diabetes Technology: Standards of Medical Care in Diabetes —2022 . Diabetes Care 2022 ; 45 ( Supplement_1 ): S97 – S112 ; doi: 10.2337/dc22-S007 . OpenUrl CrossRef PubMed 6. Andrès E , Service de Médecine Interne , Diabète et Maladies Métaboliques de la Clinique Médicale B , Hôpitaux Universitaires de Strasbourg, 1, porte de l’Hôpital, 67091 Strasbourg cedex France, Equipe de recherche EA 3072 «Mitochondrie, Stress oxydant et Protection musculaire», Faculté de Médecine de Strasbourg, Université de Strasbourg (Unistra), Strasbourg, France , et al. Telemonitoring in diabetes: evolution of concepts and technologies, with a focus on results of the more recent studies . J Med Life 2019 ; 12 ( 3 ): 203 – 214 ; doi: 10.25122/jml-2019-0006 . OpenUrl CrossRef PubMed 7. ↵ Crossen SS , Romero CC , Lewis C , et al. Remote glucose monitoring is feasible for patients and providers using a commercially available population health platform . Front Endocrinol 2023 ; 14 : 1063290 ; doi: 10.3389/fendo.2023.1063290 . OpenUrl CrossRef 8. ↵ Gaudillère M , Pollin-Javon C , Brunot S , et al. Effects of remote care of patients with poorly controlled type 1 diabetes included in an experimental telemonitoring programme . Diabetes Metab 2021 ; 47 ( 6 ): 101251 ; doi: 10.1016/j.diabet.2021.101251 . OpenUrl CrossRef PubMed 9. Crossen S , Romero C , Reggiardo A , et al. Feasibility and Impact of Remote Glucose Monitoring Among Patients With Newly Diagnosed Type 1 Diabetes: Single-Center Pilot Study . JMIR Diabetes 2022 ; 7 ( 1 ): e33639 ; doi: 10.2196/33639 . OpenUrl CrossRef 10. ↵ Prahalad P , Ding VY , Zaharieva DP , et al. Teamwork, Targets, Technology, and Tight Control in Newly Diagnosed Type 1 Diabetes: the Pilot 4T Study . J Clin Endocrinol Metab 2022 ; 107 ( 4 ): 998 – 1008 ; doi: 10.1210/clinem/dgab859 . OpenUrl CrossRef PubMed 11. ↵ Prahalad P , Tanenbaum M , Hood K , et al. Diabetes technology: improving care, improving patient reported outcomes and preventing complications in young people with Type 1 diabetes . Diabet Med 2018 ; 35 ( 4 ): 419 – 429 ; doi: 10.1111/dme.13588 . OpenUrl CrossRef PubMed 12. ↵ Prahalad , P. , Scheinker , D. , Desai , M. et al. Equitable implementation of a precision digital health program for glucose management in individuals with newly diagnosed type 1 diabetes . Nat Med 2024 ; 30 : 2067 – 2075 . doi: 10.1038/s41591-024-02975-y OpenUrl CrossRef PubMed 13. ↵ Zaharieva DP , Senanayake R , Brown C , et al. Adding glycemic and physical activity metrics to a multimodal algorithm-enabled decision-support tool for type 1 diabetes care: Keys to implementation and opportunities . Front Endocrinol 2023 ; 13 : 1096325 ; doi: 10.3389/fendo.2022.1096325 . OpenUrl CrossRef 14. ↵ Singh N , Lawrence K , Andreadis K , et al. Developing and Scaling Remote Patient Monitoring Capacity in Ambulatory Practice . NEJM Catal 2024 ; 5 ( 6 ); doi: 10.1056/CAT.23.0417 . OpenUrl CrossRef 15. Pritchett JC , Paludo JC , Borah BJ , et al. Remote Patient Monitoring to Transform Management of Febrile Neutropenia in Cancer Patients . NEJM Catal 2024 ; 5 ( 4 ); doi: 10.1056/CAT.23.0365 . OpenUrl CrossRef 16. ↵ Graseck A , Josephs M , Dokras A , et al. THEA: Prenatal Education and Remote Blood Pressure Monitoring . NEJM Catal 2024 ; 5 ( 2 ); doi: 10.1056/CAT.23.0343 . OpenUrl CrossRef 17. ↵ Scheinker D , Gu A , Grossman J , et al. Algorithm-Enabled, Personalized Glucose Management for Type 1 Diabetes at the Population Scale: Prospective Evaluation in Clinical Practice . JMIR Diabetes 2022 ; 7 ( 2 ): e27284 ; doi: 10.2196/27284 . OpenUrl CrossRef PubMed 18. ↵ Zaharieva DP , Bishop FK , Maahs DM . Advancements and future directions in the teamwork, targets, technology, and tight control—the 4T study: improving clinical outcomes in newly diagnosed pediatric type 1 diabetes . Curr Opin Pediatr 2022 ; 34 ( 4 ): 423 – 429 ; doi: 10.1097/MOP.0000000000001140 . OpenUrl CrossRef PubMed 19. Scheinker D , Prahalad P , Johari R , et al. A New Technology-Enabled Care Model for Pediatric Type 1 Diabetes . NEJM Catal 2022 ; 3 ( 5 ); doi: 10.1056/CAT.21.0438 . OpenUrl CrossRef 20. ↵ Ferstad JO , Vallon JJ , Jun D , et al. Population-level management of type 1 diabetes via continuous glucose monitoring and algorithm-enabled patient prioritization: Precision health meets population health . Pediatr Diabetes 2021 ; 22 ( 7 ): 982 – 991 ; doi: 10.1111/pedi.13256 . OpenUrl CrossRef PubMed 21. Tanenbaum ML , Zaharieva DP , Addala A , et al. ‘Much more convenient, just as effective’: Experiences of starting continuous glucose monitoring remotely following Type 1 diabetes diagnosis . Diabet Med 2022 ; 39 ( 11 ): e14923 ; doi: 10.1111/dme.14923 . OpenUrl CrossRef PubMed 22. ↵ Leverenz JC , Leverenz B , Prahalad P , et al. Role and Perspective of Certified Diabetes Care and Education Specialists in the Development of the 4T Program . Diabetes Spectr 2024 ; 37 ( 2 ): 153 – 159 ; doi: 10.2337/ds23-0010 . OpenUrl CrossRef PubMed 23. ↵ Chang A , Gao MZ , Ferstad JO , et al. A quantitative model to ensure capacity sufficient for timely access to care in a remote patient monitoring program . Endocrinol Diabetes Metab 2023 ; 6 ( 5 ): e435 ; doi: 10.1002/edm2.435 . OpenUrl CrossRef 24. ↵ Ryan D , Burke SD , Litchman ML , et al. Competencies for Diabetes Care and Education Specialists . Diabetes Educ 2020 ; 46 ( 4 ): 384 – 397 ; doi: 10.1177/0145721720931092 . OpenUrl CrossRef PubMed 25. ↵ Battelino T , Danne T , Bergenstal RM , et al. Clinical Targets for Continuous Glucose Monitoring Data Interpretation: Recommendations From the International Consensus on Time in Range . Diabetes Care 2019 ; 42 ( 8 ): 1593 – 1603 ; doi: 10.2337/dci19-0028 . OpenUrl Abstract / FREE Full Text 26. ↵ Wright EE , Morgan K , Fu DK , et al. Time in Range: How to Measure It, How to Report It, and Its Practical Application in Clinical Decision-Making . Clin Diabetes 2020 ; 38 ( 5 ): 439 – 448 ; doi: 10.2337/cd20-0042 . OpenUrl Abstract / FREE Full Text 27. ↵ Harris MA , Wagner DV , Heywood M , et al. Youth Repeatedly Hospitalized for DKA: Proof of Concept for Novel Interventions in Children’s Healthcare (NICH) . Diabetes Care 2014 ; 37 ( 6 ): e125 – e126 ; doi: 10.2337/dc13-2232 . OpenUrl FREE Full Text 28. ↵ Kim GYK , Rostosky R , Bishop FK , et al. The adaptation of a single institution diabetes care platform into a nationally available turnkey solution . Npj Digit Med 2024 ; 7 ( 1 ): 311 ; doi: 10.1038/s41746-024-01319-x . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted February 05, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care Jamie Kurtzig , Johannes O. Ferstad , Paul Dupenloup , Dessi P. Zaharieva , David M. Maahs , Priya Prahalad , Ramesh Johari , Franziska K. Bishop , Ananta Addala , David Scheinker medRxiv 2025.02.03.25321621; doi: https://doi.org/10.1101/2025.02.03.25321621 Share This Article: Copy Citation Tools A Quantitative Framework for Evaluating the Performance of Algorithm-Directed Whole-Population Remote Patient Monitoring for Type 1 Diabetes Care Jamie Kurtzig , Johannes O. Ferstad , Paul Dupenloup , Dessi P. Zaharieva , David M. Maahs , Priya Prahalad , Ramesh Johari , Franziska K. Bishop , Ananta Addala , David Scheinker medRxiv 2025.02.03.25321621; doi: https://doi.org/10.1101/2025.02.03.25321621 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Systems and Quality Improvement Subject Areas All Articles Addiction Medicine (569) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4442) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1511) Epidemiology (15230) Forensic Medicine (30) Gastroenterology (1126) Genetic and Genomic Medicine (6610) Geriatric Medicine (668) Health Economics (998) Health Informatics (4542) Health Policy (1370) Health Systems and Quality Improvement (1613) Hematology (543) HIV/AIDS (1266) Infectious Diseases (except HIV/AIDS) (15923) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (147) Nephrology (668) Neurology (6607) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1146) Occupational and Environmental Health (957) Oncology (3338) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1693) Pharmacology and Therapeutics (692) Primary Care Research (712) Psychiatry and Clinical Psychology (5448) Public and Global Health (9238) Radiology and Imaging (2202) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (596) Sexual and Reproductive Health (714) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a01e09ccbc38739c',t:'MTc3OTgxMTcwMw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();