Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions

Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions View ORCID Profile Matthew Olckers , View ORCID Profile Alexander Lam , View ORCID Profile Lazola Makhupula , View ORCID Profile Mhlasakululeka Mvubu doi: https://doi.org/10.1101/2025.08.26.25334431 Matthew Olckers 1 Stellenbosch University , South Africa Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Matthew Olckers For correspondence: olckersm{at}sun.ac.za Alexander Lam 2 Hong Kong Polytechnic University , Hong Kong Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Alexander Lam Lazola Makhupula 3 mothers2mothers , South Africa Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Lazola Makhupula Mhlasakululeka Mvubu 3 mothers2mothers , South Africa Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Mhlasakululeka Mvubu Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Client attendance is vital for the success of HIV vertical transmission prevention programs, yet 23.4% of clients missed follow-up appointments after enrolling in a community health worker-led program (n=24,807, Aug-Dec 2022). Predicting which clients are most likely to miss appointments could enable targeted interventions to improve retention. While machine learning appears well-suited for this prediction task, its effectiveness compared to community health worker judgment remains unexplored. We evaluated three machine learning approaches—logistic regression, balanced random forest, and gradient-boosted trees—trained on client enrollment records (n=51,297 training; n=18,577 test) and compared their performance to predictions by community health workers (n=61), who possess direct client interactions and contextual insights. Machine learning models achieved modest predictive performance, with the balanced random forest showing the best accuracy (ROC AUC=0.689). Community health worker predictions similarly showed low accuracy despite their rich contextual knowledge, suggesting attendance is fundamentally difficult to predict. Qualitative insights identified complex and dynamic barriers including stigma, transportation difficulties, and competing personal commitments, often unpredictable at enrollment. These findings highlight fundamental limitations in predicting client attendance and suggest that merely accumulating additional data might not enhance predictive accuracy. Instead, resources might be better allocated to addressing systemic barriers identified by community health workers. 1 Introduction Eastern and southern Africa carries a disproportionate burden of HIV, with approximately 55 percent of people and two-thirds of children living with HIV [ 31 ]. While parent-to-child (vertical) transmission of HIV can be effectively prevented through timely initiation and adherence to antiretroviral therapy, ensuring continuous engagement in care remains a major challenge. Missed appointments and interruptions in treatment undermine the success of prevention programs and threaten progress toward eliminating new pediatric infections. To achieve positive health outcomes, providers of vertical transmission prevention (VTP) programs must ensure clients attend their appointments. If a provider knew which clients were most likely to miss appointments, they could target interventions and assistance to those clients. How can health providers predict which clients are most likely to miss their appointments? As increasing amounts of health information are collected digitally, machine learning is a natural solution to predict client attendance, but most attempts have shown only modest accuracy [ 10 ]. The poor performance of machine learning to predict client attendance raises the question whether better data and models are needed, or attendance may be fundamentally difficult to predict. We compare predictions generated by a machine learning model to predictions made by community health workers, who can gain rich, contextual, and nuanced information about their clients through direct interaction. The comparison of machine learning and community health worker predictions for the same clients provides deeper understanding of the limitations of machine learning than studying machine learning models in isolation. We study these predictions in the setting of a VTP support program operated by mothers2mothers (m2m). Community health workers, which m2m call “mentor mothers”, support women living with HIV to start and adhere to treatment during pregnancy and breastfeeding. Clients meet with mentor mothers regularly, typically monthly or quarterly depending on the client’s circumstances. During the meetings, mentor mothers record information about their clients in a purpose-built mobile phone application so m2m can track client outcomes. Studies on the m2m peer mentoring program have shown that it provides significant benefits to the mentees in terms of HIV prevention, diagnosis, and monitoring. Compared to mothers who were not enrolled in the m2m program, mentees were twice as likely to return for HIV testing (of both the mother and infant) after 6-8 weeks [ 29 ] and were significantly more likely to remain in care [ 3 , 18 ], resulting in better well-being and VTP outcomes. However, these benefits require that clients adhere to the mentoring program, motivating our goal of predicting clients who are likely to miss appointments. Related literature Since we compare two approaches for predicting client attendance—-machine learning and community health worker prediction—we contribute to two distinct research areas. First, we contribute to research that uses machine learning to predict whether clients attend health appointments. 1 Second, we contribute to research that studies the accuracy of peer and community information. Several studies have used machine learning to predict client attendance in healthcare settings, often with moderate success. A study on antenatal care attendance in Ethiopia, for instance, reported modest performance (ROC AUC 0.61–0.70) despite using rich health records [ 33 ]. In HIV programs, a rapidly growing body of work uses routine electronic records to predict missed visits or client retention. A common finding across these studies is the overwhelming predictive power of a client’s past attendance. In the United States, past missed visits were the strongest predictor of future attendance, with other clinical data adding little value [ 25 ], and even externally validated models using multi-site data achieved moderate AUCs of around 0.68, with retention history being a key feature [ 28 ]. Similarly, research in South Africa and Tanzania has shown that models using prior visit history and routine electronic data can effectively predict next-visit attendance and identify clients at high risk of disengagement [ 15 , 16 , 24 ]. To make our prediction task comparable between machine learning and the community health workers, we predict attendance using only information from a client’s first appointment, without the longitudinal histories central to prior models, yet we achieve comparable predictive performance. On using local information to improve service delivery, a recent and growing literature shows that community members can hold valuable information about their peers. Examples include accurate knowledge of peers’ willingness to pay taxes [ 4 ], interest in buying land [ 23 ], business profitability [ 17 ], farming ability [ 22 ], and ability to spread information [ 5 ]. But even if community members are shown to have accurate information in one context, we cannot assume accurate knowledge in similar contexts. For example, Indonesian community members reported accurate wealth rankings about their peers [ 2 ] but a similar study, also in Indonesia, showed limited knowledge of fellow community members’ current income or need for financial support [ 30 ]. Researchers have found similar cases of inaccurate knowledge of neighbors’ poverty and wealth in Liberia [ 6 ] and Côte d’Ivoire [ 13 ]. Inaccurate knowledge extends to other domains besides wealth and poverty: in Peru, local leaders have been shown to have inaccurate knowledge of the incidence of intimate partner violence [ 1 ]. Our study documents inaccurate local knowledge in a new domain—the likelihood of attending health appointments—and complements the machine learning literature by providing a direct comparison between community predictions and algorithmic predictions. 2 Methods This mixed-methods study examined client attendance in a VTP support program through two complementary approaches. First, we quantitatively evaluated and compared predictions of client attendance generated by machine learning models with predictions made by community health workers who had direct client interactions. Second, we conducted a thematic analysis of open-ended survey responses from community health workers to understand the underlying factors and barriers that influence client attendance. This mixed-methods approach allowed us to assess not only the predictive accuracy of different methods but also to explore why attendance may be inherently difficult to predict in this context. This research project was granted ethical approval by UNSW Sydney’s Human Research Ethics Advisory Panel on 14 March 2023 (reference HC230085). The data utilized in this study originated from two sources managed by m2m: routine client records and a dedicated survey of community health workers. For the client data, which was collected during the registration process and throughout program participation; m2m obtained informed consent from these clients for the use of their data during enrollment. For the survey data, potential participants were provided with a detailed information sheet explaining the study’s purpose, data usage, and confidentiality measures. Each participating community health worker provided informed consent for the use of their survey responses prior to participation. After these internal data collection and consent procedures were completed by m2m, a de-identified version of the dataset was shared with the external research team for analysis, consistent with the ethical approval obtained (reference HC230085). 2.1 Machine learning approach Data We used de-identified data recorded at the client’s first m2m appointment. The m2m data includes clients based in Malawi, Uganda, Lesotho, South Africa, and Kenya. The data was split chronologically for training (n=51,297 clients enrolled Jan-Oct 2021) and testing (n=18,577 clients enrolled Nov 2021-Jan 2022) to simulate real-world model deployment. This approach ensures that the model is trained on past data and then evaluated on its ability to predict outcomes for new clients who enroll in the future. By using a later time period for the test set, this method realistically assesses how the model would perform when deployed in a live setting, where it must make predictions for clients not seen during its training phase. The test set was used only to choose the best performing model, from a set of models that had already been tuned using a time series split on the training data. A separate comparison set (n=24,807 clients enrolled Aug-Dec 2022) aligned with the timing of the community health worker survey. This specific cohort of clients is crucial as they were new to both the machine learning model and the community health workers at the time of the prediction task. The model, which was trained on data from Jan-Oct 2021 (and selected based on a performance on the test set of Nov 2021-Jan 2022 data), had no prior exposure to these clients. Concurrently, community health workers were surveyed to make predictions for their most recent new clients. This design allows for a direct comparison of predictive accuracy on the same set of clients. Target Measure We target an indicator for whether a client missed their next scheduled appointment, defined as no attendance record within ±7 days of the scheduled date. For instance, if a client’s appointment was scheduled for November 15, the model would check for any attendance record for that client between November 8 and November 22. If the client visited the clinic on any day within this 15-day window, they would be marked as having attended. Conversely, if no visit was recorded during this entire period, they would be classified as having missed the appointment. We focus on the first follow-up appointment after initial enrollment because it provides a consistent prediction task that is directly comparable between machine learning models and community health workers. At this stage, predictions can only draw on information collected during the enrollment visit, ensuring that neither approach relies on past attendance history, which is unavailable for new clients. By concentrating on the first follow-up, we capture a moment when targeted interventions can still be offered early in the client’s care journey. Non-attendance rates were 25.9% (training), 26.8% (test), and 23.4% (comparison set). Figure 1 illustrates non-attendance rates over time, by plotting the share of clients that missed their first follow-up appointment in each month for which we have data. The share lies between 20 and 30 percent in most months. We had access only to the data necessary to train and test the machine-learning models (Jan 2021 - Jan 2022) and the data necessary to compare the machine-learning and community health worker predictions (Aug-Dec 2022), so there is a break in series between February and July 2022. Download figure Open in new tab Fig. 1. Share of clients that missed their first follow-up appointment Features Input features for machine learning models were restricted to data available at the initial enrollment meeting ( Table 1 ), including demographics, program details, contact permissions, and basic HIV-related information. Data was captured by the community health workers using a mobile device. View this table: View inline View popup Download powerpoint Table 1. Selected Features Models We evaluated regularized logistic regression, balanced random forests [ 11 ], and several algorithms that use gradient boosted trees, namely XGBoost [ 12 ], CatBoost [ 27 ], and LightGBM [ 20 ]. Hyperparameters were tuned using five-fold time-series cross-validation on the training set, optimizing for ROC AUC. The best-performing model on the held-out test set was selected. 2.2 Community health worker predictions To compare the machine learning predictions, we surveyed m2m community health workers between August and December 2022. The survey included two types of questions designed to capture community health workers’ predictions about client attendance. First, survey respondents were asked about their most recent new client and how likely they thought that client was to miss the next appointment. Second, they were asked to nominate, from all the new clients they had seen during the week of the survey, the one they believed was most likely to miss their next appointment. For the most recent new client questions, the survey asked: “Think of your most recent new client. A new client is a client who is accessing m2m services for the first time. Please enter their first name, last name, and date of birth.” The survey then asked “Do you think {recent client name} will miss their next appointment?” and responses were recorded using a five-point Likert scale (‘definitely not’, ‘probably not’, ‘might or might not’, ‘probably yes’, ‘definitely yes’). To assess whether higher community health worker Likert ratings (“definitely not” to “definitely yes”) were monotonically associated with a higher probability of missing the next appointment, we used the Cochran–Armitage linear-by-linear trend test. Because the matched Likert sample was small (n=61) and the upper categories were sparse (including zero misses in the highest categories), we computed a Monte Carlo permutation p-value conditional on the margins (fixed group sizes and total number of misses), which is valid under sparsity and avoids reliance on large-sample normal approximations. The alternative was prespecified as one-sided (increasing), reflecting the hypothesis that higher community health worker concern implies greater miss risk. The one sided test increases statistical power. The nomination questions asked “How many new clients did you see this week?” and then followed with “Of the new clients you saw this week, which client do you think is most likely to miss their next appointment? Please enter their first name, last name, and date of birth.” To compare missing rates between clients nominated by community health workers and those not nominated, while accounting for differences across workers, we analyzed 2 × 2 tables stratified by community health worker (nominated vs. not nominated × missed vs. attended). First, we combined the stratum-specific tables to estimate a single (“common”) odds ratio using the Mantel–Haenszel method. Next, we used the Cochran–Mantel–Haenszel (CMH) chi-square test to assess whether this pooled odds ratio differed from 1 (no association). Because the CMH test conditions on each stratum’s fixed row and column totals, strata with no within-stratum variation in exposure or outcome contribute no information and are effectively dropped. We also assessed whether effects varied across strata using the Breslow–Day test. 2.3 Qualitative data collection and analysis The survey collected open-ended responses to two questions. Firstly, after the community health worker rated if they thought their most recent new client would miss their next appointment, the survey asked: “What factors might prevent {recent client name} from attending their appointments?” Secondly, the final question of the survey asked: “You have reached the end of the survey. If you like, please share any thoughts or suggestions with the research team.” Although qualitative data is most commonly associated with interviews and focus groups with a small number of respondents, online surveys have the advantage of more easily reaching a large number of respondents, which allows for a ‘wide-angle lens’ on a topic [ 9 ]. We received 168 responses to the first open-ended question and 81 responses to the second. We analyzed the survey responses using reflexive thematic analysis [ 7 , 8 ], which is a qualitative research method that focuses on identifying themes and patterns within data while embracing the researcher’s insights. Rather than trying to separate the researcher’s biases and assumptions from the analysis, this approach values the researcher’s role in shaping the analysis. We have used reflexive thematic analysis to benefit from the experience of our research team. Authors 3 and 4 worked for mothers2mothers and have extensive experience designing the mentor mother program. Reflexive thematic analysis allowed us to use insights from Authors 3 and 4 to improve our understanding of the survey responses. We use the following process to generate our themes: Author 1 read the survey responses several times and assigned initial codes. He used the codes to derive candidate themes. Author 1 then met with Authors 3 and 4 for feedback on the candidate themes, the codes, and selected quotes. Authors 3 and 4 shared insights to enrich the analysis. Author 1 shared a draft write-up of the analysis with Authors 3 and 4 for further comments. After incorporating the comments, Author 1 reread the survey responses and edited the draft. 3 Results 3.1 Machine learning predictions We report the performance of the machine learning models in Table 2 . The balanced random forest model performed best with an ROC AUC of 0.689, but still demonstrated limited ability to distinguish between clients who would attend versus miss their next appointment. Performance was comparable to similar studies [ 28 , 33 ], even though these studies used many more predictors. 2 View this table: View inline View popup Download powerpoint Table 2. ROC AUC on the held-out test set by model The ROC AUC captures the probability that the model assigns a higher risk score to a randomly chosen client who misses an appointment than to one who attends. A value of 0.5 corresponds to random guessing, while 1.0 indicates perfect classification. An AUC of 0.689 therefore means that in 68.9% of such pairs, the model correctly ranks the non-attending client higher than the attending client. Although this performance is clearly above chance, it also reveals substantial overlap in predicted risk scores between attenders and non-attenders, limiting the usefulness of the model for precise targeting. Because this analysis is exploratory, m2m has not specified a particular classification threshold. Any threshold would need to be chosen with reference to the type and cost of the intervention being considered. In the absence of a defined intervention strategy, our aim here is simply to assess whether predictions based on enrollment data could, in principle, reach levels of accuracy that would justify targeted outreach. 3.2 Community health worker predictions Each community health worker used a five-point Likert scale to record the likelihood of their client missing their next appointment. In total, 188 survey responses were collected, with the distribution across categories shown below: definitely not (71) probably not (50) might or might not (44) probably yes (16) definitely yes (7) Even though 23.4 percent of clients missed their first appointment in the comparison set (which corresponds to the same period as the survey responses), only 12.2 percent of community health worker forecasts fell into the “probably yes” or “definitely yes” categories. However, if the middle category (“might or might not”) is interpreted as a 50% probability of missing, then the weighted average forecast across all 188 responses is 23.9%, remarkably close to the observed base rate. This suggests that while few workers were willing to classify clients as high risk, their aggregate predictions were reasonably well calibrated. However, calibration to the base rate provides only a partial picture; assessing predictive accuracy requires linking survey responses to client attendance records. To test the accuracy of the community health workers’ predictions, we needed to match the name and date of birth of the client mentioned in the survey to m2m’s administrative data. Due to differences in spelling and the large number of clients with similar names, we were only able to match 61 clients to their administrative records. We took a conservative approach to ensure that, among the matched set, the attendance records corresponded to the correct client. The downside of this conservative approach is that our sample size was reduced from 188 to 61. We adopted this conservative matching strategy to minimize the risk of linking a community health worker’s response to the wrong client record. Incorrect matches would artificially bias the predictive accuracy of community health worker judgments downwards, since outcomes would be evaluated against the wrong client. Although this approach reduced our matched sample size, it maximized the validity of the comparison by ensuring that only confidently linked responses were included. In practice, integrating such prediction questions directly into the mobile health application used by community health workers would remove the need for post-hoc matching and avoid these trade-offs, but developing and deploying new application features was beyond the scope of this project. Despite the smaller sample, the matched dataset provided enough observations to test whether community health worker judgments showed very high levels of predictive accuracy. However, we lack strong statistical power to detect more modest associations, so these results should be interpreted as indicative rather than definitive. The share of clients who did miss their appointment, grouped by the five survey responses, is shown below. We also report the frequency of the response in brackets. definitely not (30): 10.0% probably not (19): 31.6% might or might not (9): 33.3% probably yes (2): 0% definitely yes (1): 0% Of the community health workers in the matched set who selected that their client would definitely not miss their next appointment, 10% did miss their appointment. As soon as the community health worker indicated some uncertainty, by selecting ‘probably not’ or ‘might or might not’, the share of missing clients increases to just over 30 percent. Only three community health workers in the matched set selected ‘probably yes’ or ‘definitely yes’, and none of these three clients missed their next appointment. Applied to the five-category Likert ratings, the Cochran–Armitage trend test showed a positive but non-significant association between higher community health worker concern and non-attendance: the permutation (Monte Carlo) p-value was 0.205 (one-sided, increasing), and the asymptotic linear-by-linear statistic yielded Z=0.954 with a corresponding one-sided p=0.170. We therefore do not reject the null of equal miss probabilities across Likert categories. The lack of statistical significance is consistent with the descriptive category counts—particularly the rarity of the highest categories and the absence of events in them, which limit power to detect a monotonic trend. The survey also asked: Of the new clients you saw this week, which client do you think is most likely to miss their next appointment? We compare attendance rates for clients picked by the community health workers against the clients they did not pick. We matched 13 community health workers between the survey and administrative data who answered this question. We could only match a small number of community health workers because we needed to match both the username of the community health worker and the name of the client they nominated. The administrative data showed that these 13 community health workers saw a further 60 clients that they did not pick in the same week. For the 13 clients the community health workers picked, 15.4% missed their next appointment. For the 60 clients the community health workers did not pick, 15.0% missed their next appointment. For 7 of the 13 community health workers, none of their clients missed their next appointment in the matched week. Therefore, these strata contained no within-community health worker outcome variation and were excluded by the conditioning step of the Cochran–Mantel–Haenszel (CMH) test. Among the remaining 6 informative strata, the CMH test provided no evidence that nomination status was associated with missing the next appointment (CMH χ 2 = 0.061, p = 0.8045). The stratum-adjusted common odds ratio comparing nominated to non-nominated clients was 0.794 (95% CI 0.129–4.909), with a Breslow–Day test indicating no meaningful heterogeneity across workers ( χ 2 = 6.940, p = 0.2251). These results are consistent with the similar raw miss rates reported above (15.4% vs. 15.0%). 3.3 Comparison between machine learning and community health worker predictions We can compare the accuracy of the machine learning predictions to the community health worker predictions for the 61 clients which the community health workers rated on the Likert scale, and for the set of 73 clients we linked to survey responses from 13 community health workers who were asked to pick which of their recent clients was most likely to miss their next appointment. Each method could be used by m2m to create a targeting rule. For the machine learning approach, m2m could target the top k clients with the highest predicted likelihood of missing the next appointment. For the Likert survey question, m2m could target the clients who were above a threshold in the response. For example, they could target clients who the community health worker reports will ‘probably’ or ‘definitely’ miss their next appointment. We can compare machine learning and community health worker predictions by choosing each rule so that they target the same number of clients. In Table 3 , we simulate a targeting rule that targets a client if the community health worker reports that their likelihood of missing their next appointment is above a threshold. For example, in the third row of Table 3 , the threshold is ‘might or might not’, so the rule would target clients for whom the community health worker reports as ‘might or might not’, probably yes’, or ‘definitely yes’ to miss their next appointment. In our sample of 61 clients, 12 clients would satisfy this rule. To compare the community health worker rule to a corresponding rule for machine learning, we take the top k clients with the highest predicted score from the machine learning model, where k is set to match the number of clients selected by the community health worker rule. So, for the third row, k = 12. Table 3 shows that the machine learning model has higher accuracy for predicting which clients will miss their next appointment. View this table: View inline View popup Download powerpoint Table 3. Comparison between machine learning (ML) and community health worker (CHW) predictions We can perform a similar exercise to compare the question where the community health worker was asked to pick a client. Two of the 13 clients the community health workers picked (from a set of 73 clients) missed their next appointment. Among the top 13 clients (from this same set of 73 clients) with the largest scores predicted by the machine learning model, three missed their next appointment. Although the machine learning approach was more accurate than the community health worker predictions, the difference is small. 3.4 Qualitative analysis of open-ended survey questions Analysis of the open-ended survey responses revealed four main themes explaining client non-attendance: reluctance to attend, competing commitments, transport difficulties, and attending at alternative times or locations. Reluctance to Attend : Some clients did not wish to attend appointments due to HIV-related stigma, concerns about medication availability at facilities, or the need for food before taking medication. Stigma was a particular concern for newly diagnosed clients; as one community health worker noted, her client might not attend because of “stigma because she’s newly tested in this month”. Lack of food could also lead to non-attendance, with one respondent explaining, “3 out of 10 clients in my caseload feels that they can’t take their drugs without food hence they opt to default on care”. Competing Commitments : Clients often faced other important obligations that clashed with appointments. These included childcare responsibilities, work or study schedules, travel for work, and attending funerals. Work conflicts were highlighted: one client had “tight schedules e.g. work and not been given permission to attend her clinics”, while another “is a young mother who is still studying at high school. She might miss appointments because she might be busy at school. Nobody to take infant to facility”. Work-related travel could also interfere: one client is “a businesswoman and travels to the nearby country Mozambique and mostly misses appointments”. Transport Barriers : Transport was the most frequently cited barrier. Challenges included distance, cost, lack of public transport, weather conditions, and reliance on partners for transport or funds. Lack of partner support could also be a significant issue: “The husband does not allow the wife to visit the health facility in general. For this time, she had to escape to come for the antenatal service”. Some clients also intentionally chose distant facilities due to stigma, compounding transport issues. Attending at Different Times or Locations : Non-attendance at a scheduled m2m appointment did not always mean missing care altogether. Clients sometimes visit the clinic at a different time than scheduled to combine visits with partners or to coincide with other errands like collecting social grants. For instance, one client “has different days for child’s CPT and her refill dates hence might miss one to combine the two”, and another “waits for husband’s date of refill to come back to the clinic”. Clients might also attend other, non-m2m facilities, sometimes due to being walk-ins or traveling for work. As respondents noted: “Most of the clients who mostly misses appointment are walk-in from different facilities”, and another client “works in Kenya. And takes a lot of months to return, but she updates us when she gets refills from that side”. These qualitative insights highlight factors, such as transport access or unexpected events like funerals, that are often dynamic, difficult to capture in administrative data available at enrollment, and thus hard to predict using either machine learning or community health worker judgment. 4 Discussion This study highlights the difficulty in predicting client attendance at a VTP program based on initial enrollment data. Machine learning models showed modest performance (ROC AUC 0.689). Community health workers, who may possess rich contextual knowledge from client interactions, were also unable to predict attendance accurately. The comparison between machine learning and community health worker predictions is a key contribution. The poor performance of community health workers suggests that the challenge lies not merely in the limitations of the features available to the machine learning algorithm, but in the fundamental unpredictability of the behavior itself in this setting. 3 The information needed to predict attendance may be unknowable at the time of prediction. Our qualitative findings corroborate this interpretation. Community health workers described a multitude of reasons for non-attendance related to stigma [ 26 ], competing commitments (work, childcare, funerals), transport barriers (cost, availability, weather, partner support), and complex healthcare navigation choices. 4 Many of these factors are volatile, arise unexpectedly after enrollment, or depend on circumstances (like food insecurity or partner disapproval) that are difficult for community health workers, let alone algorithms, to ascertain reliably. This inherent complexity and dynamism likely places an upper bound on the achievable accuracy of any predictive model based on health care records. While machine learning can identify some patterns associated with non-attendance, its utility for reliably targeting interventions in this VTP program appears limited. The modest accuracy achieved means that any targeting strategy based on these predictions would likely result in many false positives (intervening unnecessarily) and false negatives (missing clients who do need support). These findings caution against an over-reliance on predictive analytics in complex public health settings. The assumption that better data or more sophisticated algorithms will inevitably lead to effective prediction may not hold when the target behavior is driven by highly dynamic, contextual, and often unobservable factors. Ethical considerations also arise; deploying potentially inaccurate or biased predictive models for targeting could lead to inefficient resource allocation, stigmatization, or failure to address the true, diverse needs of the client population [ 32 ]. Instead of focusing primarily on prediction, our results suggest that resources may be more effectively directed towards understanding and mitigating the systemic barriers to attendance identified through qualitative insights from community health workers and clients. Addressing issues like transport costs, inflexible clinic hours, stigma reduction, and ensuring consistent service quality could potentially improve attendance more broadly and equitably than relying on imperfect predictive targeting. A Competing Interests LM and MM were employed by mothers2mothers, the organization whose program data was used in this study. MO and AL declare no competing interests. B Authors’ contributions MO designed the study and survey instrument in consultation with LM and MM. With support from their colleagues at m2m, LM and MM organized the health records used for machine learning and conducted the survey. MO and AL conducted quantitative data analysis and model training. MO completed the analysis of the qualitative data with input from LM and MM. MO wrote the manuscript with input from AL, LM, and MM. C Funding The research team and mothers2mothers received funding from the Google AI for Social Good Program for 2021-22. D Data Availability Statement The data that supports the findings of this study are available from mothers2mothers. Restrictions apply to the availability of these data, which were used under license for this study. Data are available from the author(s) with the permission of mothers2mothers. The code used to replicate the results is available in the following repository: https://github.com/matthewolckers/m2m-predict-attendance E Hyperparameters View this table: View inline View popup Download powerpoint Table 4. Selected hyperparameters for best models (library parameter names) Acknowledgments We thank the mothers2mothers community health workers who participated in the survey and shared their valuable insights. We thank Mohammed Fadul, Garrit Gerke, Nakululombe Kwendeni, Raymond Moholisa, Moeti Moleko, and Robynn Paulsen-Naidoo from m2m for their assistance in preparing the data and their advice on m2m’s systems. We thank Tuba Islam from Google for her advice on machine learning approaches. We thank Brendan Maughan-Brown, Bryan Wilder and Lily Xu for helpful comments. Footnotes Authors’ Contact Information: Matthew Olckers, olckersm{at}sun.ac.za , Stellenbosch University, Stellenbosch, South Africa; Alexander Lam, Hong Kong Polytechnic University, Hong Kong, alexander-a.lam{at}polyu.edu.hk ; Lazola Makhupula, mothers2mothers, Cape Town, South Africa; Mhlasakululeka Mvubu, mothers2mothers, Cape Town, South Africa. CCS Concepts:•Applied computing→Health informatics;•Computing methodologies→Classification and regression trees; Supervised learning by classification;•Human-centered computing→Field studies. ↵ 1 We focus on studies that predict appointment attendance or retention in care. A related literature predicts medication adherence [ 19 , 21 ]. ↵ 2 One recent study reported an ROC AUC of 0.895 [ 14 ], but the study does not explain whether a held-out test set was used to measure performance. Measuring performance on the training data can inflate performance measures. ↵ 3 An alternative interpretation is that community health workers observed relevant information but this information was not effectively translated into accurate predictions due to the way the survey elicited forecasts. As this is, to our knowledge, the first study to elicit predictions of client attendance, we encourage future research to explore alternative elicitation methods in larger samples. ↵ 4 These barriers to attendance have also been reported in other contexts. For example, a qualitative study with HIV-infected adults in the United States reported stigma, unreliable transport, and competing life activities among the reasons for the non-attendance [ 34 ]. References [1]. ↵ Jorge M Agüero , Úrsula Aldana , Erica Field , Veronica Frisancho , and Javier Romero . 2020 . Is Community-Based Targeting Effective in Identifying Intimate Partner Violence? AEA Papers and Proceedings 110 ( 2020 ), 605 – 609 . OpenUrl [2]. ↵ Vivi Alatas , Abhijit Banerjee , Arun G Chandrasekhar , Rema Hanna , and Benjamin A Olken . 2016 . Network structure and the aggregation of information: Theory and evidence from Indonesia . American Economic Review 106 , 7 (2016), 1663 – 1704 . OpenUrl CrossRef [3]. ↵ Carolyn Baek , Vuyiswa Mathambo , Sibongile Mkhize , Irwin Friedman , Louis F. Apicella , and Naomi Rutenberg . 2007 . Key findings from an evaluation of the mothers2mothers program in KwaZulu-Natal South Africa . Technical Report. Horizons Final Report . [4]. ↵ Pablo Balán , Augustin Bergeron , Gabriel Tourek , and Jonathan L Weigel . 2022 . Local elites as state capacity: How city chiefs use local information to increase tax compliance in the democratic republic of the Congo . American Economic Review 112 , 3 (2022), 762 – 797 . OpenUrl [5]. ↵ Abhijit Banerjee , Arun G Chandrasekhar , Esther Duflo , and Matthew O Jackson . 2019 . Using Gossips to Spread Information: Theory and Evidence from Two Randomized Controlled Trials . The Review of Economic Studies 86 , 6 (2019), 2453 – 2490 . OpenUrl [6]. ↵ Lori Beaman , Niall Keleher , Jeremy Magruder , and Carly Trachtman . 2021 . Urban Networks and Targeting: Evidence from Liberia . AEA Papers and Proceedings 111 ( 2021 ), 572 – 76 . OpenUrl [7]. ↵ Virginia Braun and Victoria Clarke . 2006 . Using thematic analysis in psychology . Qualitative research in psychology 3 , 2 (2006), 77 – 101 . OpenUrl CrossRef [8]. ↵ Virginia Braun and Victoria Clarke . 2022 . Thematic Analysis: A Practical Guide . SAGE Publications Ltd , London, UK . [9]. ↵ Virginia Braun , Victoria Clarke , Elicia Boulton , Louise Davey , and Charlotte McEvoy . 2021 . The online survey as a qualitative research tool . International journal of social research methodology 24 , 6 (2021), 641 – 654 . OpenUrl [10]. ↵ Danae Carreras-García , David Delgado-Gómez , Fernando Llorente-Fernández , and Ana Arribas-Gil . 2020 . Patient no-show prediction: a systematic literature review . Entropy 22 , 6 (2020), 675 . OpenUrl PubMed [11]. ↵ Chao Chen , Andy Liaw , Leo Breiman , et al. 2004 . Using random forest to learn imbalanced data. (2004) , 24 pages. Working Paper. [12]. ↵ Tianqi Chen and Carlos Guestrin . 2016 . Xgboost: A scalable tree boosting system . In Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery & Data Mining . Association for Computing Machinery , New York, NY, USA , 785 – 794 . [13]. ↵ Pascaline Dupas , Marcel Fafchamps , and Deivy Houeix . 2022 . Measuring Relative Poverty Through Peer Rankings: Evidence from Côte d’Ivoire. (2022). Working Paper . [14]. ↵ Tamrat Endebu , Girma Taye , and Wakgari Deressa . 2025 . Development of a machine learning prediction model for loss to follow-up in HIV care using routine electronic medical records in a low-resource setting . BMC Medical Informatics and Decision Making 25 , 1 (2025), 192 . OpenUrl [15]. ↵ Rachel T Esra , Jacques Carstens , Janne Estill , Ricky Stoch , Sue Le Roux , Tonderai Mabuto , Michael Eisenstein , Olivia Keiser , Mhari Maskew , Matthew P Fox , et al. 2023 . Historical visit attendance as predictor of treatment interruption in South African HIV patients: extension of a validated machine learning model . PLOS Global Public Health 3 , 7 (2023), e0002105 . OpenUrl [16]. ↵ Carolyn A Fahey , Linqing Wei , Prosper F Njau , Siraji Shabani , Sylvester Kwilasa , Werner Maokola , Laura Packel , Zeyu Zheng , Jingshen Wang , and Sandra I McCoy . 2022 . Machine learning with routine electronic medical record data to identify people at high risk of disengagement from HIV care in Tanzania . PLOS Global Public Health 2 , 9 (2022), e0000720 . OpenUrl PubMed [17]. ↵ Reshmaan Hussam , Natalia Rigol , and Benjamin N Roth . 2022 . Targeting high ability entrepreneurs using community information: Mechanism design in the field . American Economic Review 112 , 3 (2022), 861 – 98 . OpenUrl [18]. ↵ Jude Ofuzinim Igumbor , Joseph Ouma , Kennedy Otwombe , Eustasius Musenge , Felix Chima Anyanwu , Tariro Basera , Marjorie Mbule , Esca Scheepers , and Kathrin Schmitz . 2019 . Effect of a Mentor Mother Programme on retention of mother-baby pairs in HIV care: A secondary analysis of programme data in Uganda . PloS one 14 , 10 (2019), e0223332 . OpenUrl PubMed [19]. ↵ Przemyslaw Kardas , Pawel Lewek , and Michal Matyjaszczyk . 2013 . Determinants of patient adherence: a review of systematic reviews . Frontiers in pharmacology 4 ( 2013 ), 91 . OpenUrl PubMed [20]. ↵ Guolin Ke , Qi Meng , Thomas Finley , Taifeng Wang , Wei Chen , Weidong Ma , Qiwei Ye , and Tie-Yan Liu . 2017 . LightGBM: A Highly Efficient Gradient Boosting Decision Tree . In Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, December 4–9, 2017, Long Beach, CA, USA . Curran Associates, Inc ., NY, USA , 3146 – 3154 . [21]. ↵ Jackson A. Killian , Bryan Wilder , Amit Sharma , Vinod Choudhary , Bistra Dilkina , and Milind Tambe . 2019 . Learning to Prescribe Interventions for Tuberculosis Patients Using Digital Adherence Data . In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (Anchorage, AK, USA) (KDD ‘19) . Association for Computing Machinery , New York, NY, USA , 2430 – 2438 . doi: 10.1145/3292500.3330777 OpenUrl CrossRef [22]. ↵ Pushkar Maitra , Sandip Mitra , Dilip Mookherjee , Alberto Motta , and Sujata Visaria . 2017 . Financing Smallholder Agriculture: An Experiment with Agent-Intermediated Microloans in India . Journal of Development Economics 127 ( 2017 ), 306 – 337 . OpenUrl [23]. ↵ Martina Manara and Tanner Regan . 2025 . Ask a local: Improving the public pricing of land titles in urban Tanzania . Review of Economics and Statistics 107 , 1 (2025), 188 – 203 . OpenUrl [24]. ↵ Mhairi Maskew , Kieran Sharpey-Schafer , Lucien De Voux , Thomas Crompton , Jacob Bor , Marcus Rennick , Admire Chirowodza , Jacqui Miot , Seithati Molefi , Chuka Onaga , et al. 2022 . Applying machine learning and predictive modeling to retention and viral suppression in South African HIV treatment cohorts . Scientific reports 12 , 1 (2022), 12715 . OpenUrl PubMed [25]. ↵ Brian W Pence , Angela M Bengtson , Stephen Boswell , Katerina A Christopoulos , Heidi M Crane , Elvin Geng , Jeanne C Keruly , W Christopher Mathews , and Michael J Mugavero . 2019 . Who will show? Predicting missed visits among patients in routine HIV primary care in the United States . AIDS and Behavior 23 , 2 (2019), 418 – 426 . OpenUrl PubMed [26]. ↵ Teodora Perger , Mariam Davtyan , Caroline Foster , Michael Evangeli , Claire Berman , Deborah Kacanek , Ana M Puga , Serufusa Sekidde , and Sanj Bhopal . 2025 . Impact of HIV-related stigma on antiretroviral therapy adherence, engagement and retention in HIV care, and transition to adult HIV Care in Pediatric and Young Adult Populations Living with HIV: a literature review . AIDS and Behavior 29 , 2 (2025), 497 – 516 . OpenUrl PubMed [27]. ↵ Liudmila Ostroumova Prokhorenkova , Gleb Gusev , Aleksandr Vorobev , Anna Veronika Dorogush , and Andrey Gulin . 2018 . CatBoost: unbiased boosting with categorical features . In Advances in Neural Information Processing Systems 31: Annual Conference on Neural Information Processing Systems 2018, NeurIPS 2018, December 3-8, 2018, Montréal, Canada . Curran Associates, Inc ., NY, USA , 6639 – 6649 . [28]. ↵ Jessica P Ridgway , Aswathy Ajith , Eleanor E Friedman , Michael J Mugavero , Mari M Kitahata , Heidi M Crane , Richard D Moore , Allison Webel , Edward R Cachay , Katerina A Christopoulos , et al. 2022 . Multicenter development and validation of a model for predicting retention in care among people with HIV . AIDS and Behavior 26 , 10 (2022), 3279 – 3288 . OpenUrl PubMed [29]. ↵ Amir Shroufi , Emma Mafara , Jean François Saint-Sauveur , Fabian Taziwa , and Mari Carmen Viñoles . 2013 . Mother to mother (M2M) peer support for women in prevention of mother to child transmission (PMTCT) programmes: a qualitative study . PloS one 8 , 6 (2013), e64717 . OpenUrl CrossRef PubMed [30]. ↵ Carly Trachtman, Yudistira Hendra Permana, and Gumilang Aryo Sahadewo . 2021 . How Much Do Our Neighbors Really Know? The Limits of Community-Based Targeting. (2021). Working Paper . [31]. ↵ UNAIDS . 2021 . hUNAIDS Data 2021. Page 44 . [32]. ↵ Angelina Wang , Sayash Kapoor , Solon Barocas , and Arvind Narayanan . 2023 . Against predictive optimization: On the legitimacy of decision-making algorithms that optimize predictive accuracy . ACM Journal on Responsible Computing 1 , 1 , Article 9 (2023), 45 pages. OpenUrl [33]. ↵ Bryan Wilder , Clara Pons-Duran , Frederick GB Goddard , Bezawit Mesfin Hunegnaw , Sebastien Haneuse , Delayehu Bekele , and Grace J Chan . 2023 . Development of prediction models for antenatal care attendance in Amhara region, Ethiopia . JAMA Network Open 6 , 5 (2023), e2315985 – e2315985 . OpenUrl [34]. ↵ Baligh R Yehia , Leslie Stewart , Florence Momplaisir , Aaloke Mody , Carol W Holtzman , Lisa M Jacobs , Janet Hines , Karam Mounzer , Karen Glanz , Joshua P Metlay , et al. 2015 . Barriers and facilitators to patient retention in HIV care . BMC infectious diseases 15 , 1 (2015), 246 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted August 27, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions Matthew Olckers , Alexander Lam , Lazola Makhupula , Mhlasakululeka Mvubu medRxiv 2025.08.26.25334431; doi: https://doi.org/10.1101/2025.08.26.25334431 Share This Article: Copy Citation Tools Limited Predictability of Client Attendance in a Support Program for HIV Vertical Transmission Prevention: A Comparison of Machine Learning and Community Health Worker Predictions Matthew Olckers , Alexander Lam , Lazola Makhupula , Mhlasakululeka Mvubu medRxiv 2025.08.26.25334431; doi: https://doi.org/10.1101/2025.08.26.25334431 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4426) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15222) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6589) Geriatric Medicine (667) Health Economics (997) Health Informatics (4525) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (971) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9221) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (711) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffec462af3d52ad',t:'MTc3OTQ4MzgwMg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();