Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment

Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment View ORCID Profile Andrea Kusec , View ORCID Profile Kym IE Snell , View ORCID Profile Nele Demeyere doi: https://doi.org/10.1101/2024.09.06.24313196 Andrea Kusec 1 Nuffield Department of Clinical Neurosciences, University of Oxford , Oxford, UK PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Andrea Kusec Kym IE Snell 2 Department of Applied Health Sciences, School of Health Sciences, University of Birmingham , Birmingham, UK 3 National Institute for Health and Care Research (NIHR) Birmingham Biomedical Research Centre , UK PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kym IE Snell Nele Demeyere 1 Nuffield Department of Clinical Neurosciences, University of Oxford , Oxford, UK PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Nele Demeyere For correspondence: nele.demeyere{at}ndcn.ox.ac.uk Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Background and Objectives Post-stroke cognitive impairment (PSCI) is highly prevalent across multiple cognitive domains. Individualised PSCI prognosis has mainly been researched using outcomes not developed specifically for stroke (i.e., dementia risk). Further, existing models often use predictors not routinely available in electronic health records. Here, we develop and externally validate clinical prediction models for overall PSCI using a stroke-specific cognitive outcome using acute cognition and data routinely collected as part of stroke care. Methods N =430 stroke survivors completed the Oxford Cognitive Screen (OCS) in acute care and at 6-month follow-up (binarized outcome; impaired vs unimpaired). Logistic regression models were fitted comprising both mandatory clinically-relevant (age, sex, stroke severity, education, stroke hemisphere, acute PSCI) and data-driven (acute mood difficulties, length of stay in acute care, multimorbidity) predictors using backward elimination ( p < 0.10) on multiply imputed data. Internal validation used bootstrapping to obtain optimism-adjusted performance estimates. External validation used the optimism-adjusted C-Slope as a uniform shrinkage factor. Results The overall PSCI model demonstrated good optimism-adjusted performance (C-Statistic=0.76 [95% CI=0.71–0.80]). In external validation, the overall PSCI model was comparable to development data (C-Statistic=0.74 [95% CI=0.67–0.79]). PSCI model performance did not vary by sex, but performed best in adults <60 years old (C-Statistic=0.76) with moderate-severe acute PSCI (C-Statistic=0.72). We further explored modelling of Discussion Prediction models of stroke-specific cognition have the potential to offer more meaningful PSCI prognoses, including domain-specific cognitive recovery, compared to existing models focused on domain-general decline. The present model shows promise of generalisability with an initial good external validation performance of the PSCI model in a different stroke severity cohort. Future recalibration of domain-specific models would be beneficial to provide additional detailed predictions. Introduction Stroke is the leading cause of long-term physical and cognitive disability worldwide 1 . One-year post-stroke cognitive impairment (PSCI) prevalence estimates range from 40% 2 to 98% 3 , with rates of 45% 4 to 80% 5 in chronic stroke. PSCI negatively impacts patients 6 , caregivers/families 7 , and has considerable economic costs 8 . Clinical prediction models have been developed to improve PSCI prognostication, chiefly post-stroke cognitive decline and dementia 9 , 10 . However, PSCI does not necessarily cause cognitive decline or dementia. Research demonstrates that whilst some patients exhibit decline, others have a stable, chronic cognitive impairment or even demonstrate continued improvement 11 . New PSCI definitions acknowledge the complex interplay of declining brain health, focal brain injury and cognitive recovery, with outcomes including decline, stability, and improvement 12 . PSCI is highly prevalent across multiple cognitive domains of language, memory, attention, numeracy, executive function, and praxis 13 . These impairments have previously been studied in isolation, despite research demonstrating differing recovery rates across domains (e.g., hemispatial neglect 14 ). Existing prediction models of post-stroke dementia 15 – 17 perform poorly in PSCI 18 , 19 , possibly because specific cognitive domains have different relationships to functional outcomes 20 . Newer models have attempted to improve PSCI prediction, but remain focused on traditional clinical and demographic predictors (e.g., age, stroke severity). A recent meta-analysis ( N =160,783) of PSCI predictors reported that, by far, the strongest predictor of chronic PSCI was acute cognitive functioning 21 , demonstrating the importance of baseline cognitive performance in developing accurate prediction models 22 . However, existing PSCI prediction models do not routinely include acute cognition as a predictor. With early PSCI assessment now recommended by national and international guidelines 23 – 25 acute cognitive data should be routinely available and used in PSCI prognostication. Existing PSCI prediction models often assess acute and long-term cognition via dementia screening tools (e.g., Mini Mental State Examination). These tools are not suitable for left hemisphere stroke due to overreliance on verbal abilities 26 . Utilizing a stroke-specific cognitive screen for PSCI clinical prediction models would avoid assessment confounds and is more strongly associated with 6-month cognitive recovery 13 , 27 . Study Aims To develop and externally validate clinical prediction models of 6-month PSCI outcomes using acute cognitive information via a stroke-specific tool. Methods The study is a secondary analysis of data collected from the Oxford Cognitive Screening Programme. All participants provided informed consent to take part (REC Reference: 18/SC/0550). Participants Participants comprised a single, consecutively recruited cohort from the John Radcliffe Hospital acute stroke ward between March 2012 (first consented participant) to March 2020 (final follow-up participant). Programme inclusion criteria were 1) stroke diagnosis (first ever or recurrent; determined via CT/MRI scan); 2) ≥18 years; 3) ability to remain alert for ≥20 minutes, and 4) ability to provide informed consent. Participants completed a brief stroke-specific cognitive assessment acutely ( N= 866) and at 6-months post-stroke ( N= 430). Attrition from acute to 6-months was due to loss to follow-up ( n =149), participant death ( n =82), declining follow up ( n= 76), being too unwell to take part ( n =72), moved out of area ( n =36), had an incomplete assessment ( n= 16), or could not be seen due to COVID-19 restrictions ( n =5). Those lost to attrition versus those retained only differed in terms of having greater stroke severity and greater acute cognitive impairments. A detailed account of attrition is in Milosevich et al. 13 Stroke severity (National Institute of Health Stroke Severity; NIHSS) and other stroke-related details (e.g., lesion hemisphere, first vs recurrent stroke) were obtained from electronic health records. Study Outcome Measure Overall PSCI was assessed using the Oxford Cognitive Screen (OCS 28 ), a measure of stroke-specific cognition across multiple cognitive domains affected by stroke. The OCS was administered at a mean of 4.38 days ( SD = 4.46) post-stroke The OCS comprises 12 subtasks forming 6 cognitive domains: Language (picture naming, semantics, sentence reading), Memory (orientation, verbal recall, episodic recognition), Spatial attention (broken hearts cancellation task), Numeracy (number writing, calculation), Praxis (gesture imitation), and Executive Function (mixed trails). Subtask scores are binarized as impaired (1) or unimpaired (0) relative to cut-off scores from a normative sample. A domain impairment was defined as the presence of any impairment in any subtask within that domain. Models developed included overall PSCI severity (total proportion of OCS subtasks impaired at 6-months post-stroke; continuous outcome model), and binarized PSCI presence in any domain (logistic outcome model). Analysis Analyses were performed in R Version 4.4.0 29 . Baseline descriptive statistics were first summarised. R packages used included rms 30 , psfmi 31 , mice 32 , and pmvalsampsize 33 . Data is freely available at https://www.dementiasplatform.uk/ and analysis code at https://osf.io/3pc5k/ Predictor Selection We selected predictors likely to be available in electronic health records upon deployment. This included clinically relevant predictors (age at stroke, sex, NIHSS scores, education years, first vs recurrent stroke, type of stroke [ischaemic vs haemorrhagic], stroke hemisphere, and acute cognitive impairment 21 ). The predictor stroke hemisphere included an “undetermined” category where either lateralisation was not clear based on CT/MRI scan data and/or clinical presentation was not conclusive. We additionally included “data-driven predictors” that are available in electronic health records (length of stay in acute care, independence prior to admission [defined as not requiring paid or family carer support >2 hours per week], presence of mood difficulties during acute care [as reported by clinical records during stay], and Charlson Comorbidity index). Each model therefore had 13 potential predictors forming an “initial model.” Following predictor selection, performance was estimated with clinically relevant predictors and only significant data-driven predictors retained, labelled throughout the manuscript as the “final model.” Predictor selection per model was repeated using bootstrapping across 1000 iterations. Sample Size Justification Sample size sufficiency was evaluated based on the events fraction, total sample size, number of predictor parameters, and a target shrinkage factor of >0.90 to minimise overfitting 34 , 35 . Event fraction rates and assumed apparent Nagelkerke’s R 2 performance (0.30) were based on previous Oxford Cognitive Screening programme analyses 13 . For the overall PSCI model ( N Events=295 of 430), the sample size required was 393 participants, indicating sufficient power with the current data. Missing Data Management Those with complete vs incomplete data at 6-months post-stroke were compared on predictor variables. To increase statistical power and reduce bias, multiple imputation was conducted across 20 imputed datasets (due to 28.8% missingness in acute NIHSS scores) with 50 iterations. Data were assumed missing at random given that the variable with the highest missingness rate (NIHSS) was historically unavailable in electronic health records in earlier recruitment periods. Only predictor variables were imputed 36 . Upon model deployment, missingness is likely to occur (e.g., missing stroke severity information) and therefore imputation would be necessary 36 . Sensitivity analyses were conducted investigating the influence of missing information. A detailed account of participant attrition in this cohort is elsewhere 13 . Model Development and Internal Validation Across all models, clinically relevant predictors were retained irrespective of statistical significance. Backward stepwise elimination was used to remove only non-significant ( p >0.10) data-driven predictors. This approach was taken given criticisms around removing clinically relevant (though statistically insignificant) predictors 37 . Predictor selection was conducted only on data-driven predictors, as clinically relevant predictors were viewed to be mandatory to retain due previous research demonstrating associations to 6-month PSCI, their conceptual importance, and face validity for clinicians using the model upon deployment. Models developed across 20 imputed datasets were compared to complete case data. Apparent final model performance (i.e., non-significant data-driven predictors removed) was evaluated using discrimination (model’s ability to correctly identify individuals with and without 6-month PSCI; estimated via the Area Under the Curve [AUC; binary outcome models only], C-Statistic), calibration measures (calibration-in-the-large [CITL], calibration slope [C-Slope], calibration plots, Brier scores, observed-expected ratio) and goodness-of-fit measures (adjusted R 2 , continuous PSCI model; Nagelkerke’s R 2 , binary PSCI models). Pooled (across imputed datasets) b-values, odds ratios (ORs), and performance statistics are reported per model. Optimism-adjusted performance estimates were obtained via bootstrapping each model on multiply imputed data across 1000 iterations. The model-specific optimism-adjusted C-Slope was used as a uniform shrinkage factor and was multiplied with model regression coefficients to correct for potential overfitting 38 , 39 . Model intercepts were re-estimated using the shrunken regression coefficients to obtain an accurate CITL. Risk groups were created using 10 th decile groups on prediction model estimates for visualisation purposes via calibration plots. External Validation The OCS-Care dataset 11 was used for external validation. In parallel to OCS-Recovery data, the OCS-Care dataset ( N =264, M age = 68.9) assessed acute PSCI using the OCS and 6-months later, comprising a mild severity cohort ( Mean NIHSS=2.8). Model predictors were collected from electronic health records. Many OCS-Care participants had at least one cognitive impairment at 6-months ( N events=147; 55.6%), versus 70.8% acutely ( N events=187). When estimating minimum sample size requirements 40 , this dataset was sufficient to estimate a C-Statistic of 0.80 (CI width=0.20), though precise C-Slope estimates require a much larger dataset. Shrunken model coefficients obtained through internal validation were applied to the OCS-Care data to estimate performance. Performance measures described above (C-Slope, CITL, C-Statistic, R 2 and Nagelkerke’s R 2 ) were estimated. Overall binary PSCI model performance was further evaluated within subgroups by age range, sex, first vs recurrent stroke, and acute PSCI severity. Exploratory Domain-Specific Prediction Models Because the OCS provides information on domain-specific cognition on language, attention, memory, numeracy, praxis, and executive function, we additionally explored creating logistic domain-specific prediction models using the same development and validation methods detailed above for the overall PSCI model as a signal for future development of domain-specific models. Given the sample size requirements were not met for individual subdomains of Language (1000 participants; N Events=138 of 428), Memory (1016 participants; N Events=137 of 430), Attention (638 participants; N Events=187 of 416), Praxis (2912 participants; N Events=79 of 403), Number (2524 participants; N Events=85 of 420), and Executive Function (1724 participants; N Events=98 of 400) models, these results are presented in the Supplementary Materials . Results Demographics are in Table 1 . At 6-months, all participants provided outcome data. View this table: View inline View popup Download powerpoint Table 1. Participant demographics. NIHSS=National Institute of Health Stroke Severity Participants with missing 6-month PSCI data were more likely to have higher acute PSCI ( p <0.0001), be older in age ( p =0.02), and have acute language ( p <0.01), memory ( p <0.001), or numeracy impairments ( p 0.15; see Supplemental Materials ). Overall PSCI Models Pooled shrunken coefficients of the final overall continuous and binary PSCI models are in Table 2 . View this table: View inline View popup Table 2. Pooled model coefficients of continuous and binary overall PSCI models. Shrinkage was applied to coefficients using the optimism-adjusted C-Slope. For categorical variables stroke hemisphere and independence pre-admission, reference categories were left-hemisphere stroke and independent pre-admission, respectively. NIHSS=National Institute of Health Stroke Severity; OCS=Oxford Cognitive Screen In the multivariable continuous model of proportion of 6-months OCS tasks impaired, the strongest clinically relevant predictors included higher age (pooled B =0.002 [95% CI=0.001– 0.003]) and a greater proportion of acute OCS tasks impaired (pooled B =0.35 [95% CI=0.29– 0.41]). In bootstrapped and complete case data, the only data-driven predictor retained was requiring carer support prior to admission (pooled B =0.10 [95% CI=0.05–0.14]) and improved model fit in complete case data ( F =3.69, p =0.03). The optimism-adjusted performance of the continuous overall PSCI model was good to excellent (C-Slope=0.96, CITL=-0.01; MSE=0.02; Adj R 2 =0.34). In the multivariable binary model, higher age (pooled OR=0.93 [95% CI=0.92–0.93]), bilateral hemisphere lesions (pooled OR=0.75 [95% CI=0.64–0.88]), fewer years of education (pooled OR=0.91 [95% CI=0.89–0.92]), and a greater proportion of acute OCS tasks impaired (pooled OR=1.62 [95% CI=1.35–1.95]) were associated with an increased risk of 6-month PSCI. No data-driven predictors were retained for the binary PSCI model. The final optimism-adjusted performance showed good performance (C-Statistic=0.76 [95% CI=0.71–0.80]; C-Slope=0.93 [95% CI=0.75–1.11]; CITL= −1.17 [95% CI= −1.39– −0.95]; Brier Score=0.12 [95% CI=0.10–0.14]; Nagelkerke’s R 2 =0.21). In sensitivity analyses using complete case data, there were no notable differences in predictor selection for either the continuous or binary overall PSCI models. Unadjusted relationships between predictor and outcome variables for the overall PSCI models are in Table S11. All model performance measures for overall PSCI are in Table 3 . Download figure Open in new tab Figure 1. Calibration plot of any 6-month PSCI (0=no impairment, 1=any impairment) across 20 imputed datasets. Complete cases plot is in the Supplementary Figure S1 . View this table: View inline View popup Download powerpoint Table 3. Performance metrics all final models pooled across 20 imputed datasets. CITL=Calibration-in-the-large; E/O=Expected:Observed Ratio; C-Slope=Calibration Slope External Validation External validation estimates are in Table 4 . Model discrimination, calibration, and goodness-of-fit was not substantially discrepant in the OCS-Care dataset, suggesting good model performance across cohorts. View this table: View inline View popup Download powerpoint Table 4. Model performance following external validation across 20 imputed datasets using OCS-Care data. CITL=Calibration-in-the-large; C-Slope=Calibration Slope In the binary overall PSCI model subgroup analyses ( Table 5 ), performance did not vary by sex (Male C-Statistic=0.76 [0.67–0.83], Female C-Statistic=0.76 [0.66–0.84]) or by first versus recurrent stroke (First-ever C-Statistic=0.75 [0.67-0.81]; Recurrent C-Statistic=0.73 [0.57–0.84]. Model performance varied by age group (60 years C-Statistic=0.65 [0.48–0.78], >70 years C-Statistic=0.65 [0.49–0.78], >80 years C-Statistic=0.71 [0.52–0.72]), and by level of acute PSCI (Mild acute PSCI C-Statistic=0.62 [0.52–0.72]; Moderate-severe acute PSCI=0.72 [0.61–0.81]). View this table: View inline View popup Download powerpoint Table 5. Subgroup analysis of overall binary cognitive model performance in external data. PSCI = Post-stroke cognitive impairment We developed an online risk calculator of the binary overall PSCI model given its promising performance https://ocs-strokecogpredictor.shinyapps.io/OCS_Stroke_Cognition_Predictor/ allowing further independent evaluation in future research. The tool allows entry of raw values of each key predictor, resulting in a percentage likelihood of domain-specific PSCI impairment at 6-months post-stroke. Exploratory Domain-Specific Prediction Models Optimism-adjusted performance of the exploratory domain-specific models of PSCI were either similar to overall PSCI models (Language, Attention; C-Statistic range=0.73–0.77) or had lower performance (Memory, Numeracy, Executive Function, and Praxis; C-Statistic range=0.60–0.71). In external validation, Language and Attention models demonstrated good discrimination and goodness-of-fit, however calibration was poor across all domains, particularly in Memory, Numeracy, Executive Function, and Praxis. Full details of the exploratory domain-specific models, including external validation, are shown in the Supplementary Materials . Discussion We developed and externally validated overall PSCI prediction models, utilising acute cognitive information from a stroke-specific screen (OCS) alongside established PSCI predictors. To our knowledge, this is the first study to use acute cognitive data and a stroke-specific measure of PSCI in clinical prediction model development. We additionally explored creating domain-specific prediction models to be developed in future research. Overall Model Performance Our models provided good explanatory power, with optimism-adjusted C-Slopes of 0.95 and 0.76 for continuous and binary overall PSCI models, respectively. Compared to models of post-stroke decline and dementia (C-Statistic range=0.53–0.66 19 ), our models comparatively perform better even when considering domain-specific models (C-Statistic range=0.60–0.77). Promisingly, in external data, overall PSCI model performance was comparable to internal validation, suggesting it could be used across different stroke cohorts. The domain-specific PSCI model performance showed some variation within subgroups, with a better performance in younger age groups (<60) and with moderate-severe acute PSCI. This is in line with the nature of these domain-specific PSCI predictions being indicative of stroke-specific focal cognitive changes and their recovery, rather than overall, non-stroke specific brain health related cognitive outcomes (e.g. dementia/decline). In external data, CITL estimates were consistently negative, suggesting a small systematic overprediction of 6-month PSCI risk. This is likely due to development data comprising a more moderate-severe stroke cohort, whilst external data comprised a more mild stroke cohort. Additionally, C-Slopes in external data were larger, potentially indicating overshrinkage. This prediction model should further be recalibrated across a range of stroke severities. Clinical Implementation of Overall PSCI Models Age, sex, years of education, NIHSS scores, recurrent stroke and stroke type contribute to PSCI 21 and should be included in recalibrations of our models. Novel predictors should also be considered; our modelling approach includes data-driven predictors to allow for routine model updating. This approach identified potential predictors for recalibration (i.e., requiring carer support pre-stroke and comorbidity) that are currently excluded from PSCI prediction modelling 9 , 19 , 41 , 42 . Crucially, we only selected predictors available in electronic health records, in order to facilitate future implementation. Many PSCI models often include predictors not routinely available at deployment 9 , 10 , 19 . In the UK specifically, predictor selection should be guided by the National Clinical Guideline for Stroke 25 such as including acute cognitive assessment, as these are more likely to available. Other biopsychosocial (e.g., white matter hyperintensities, socioeconomic status) and clinical (e.g., amount/intensity of neurorehabilitation offered) predictors may improve model performance, however these may be less available or have significant economic considerations. For example, imaging-based data improves PSCI prediction models 43 , however behavioural data (e.g., cognitive assessments) is considerably more affordable and feasible to implement 44 , 45 . As typical for electronic health record data, NIHSS scores had large amounts of missingness. Imputation methods should be considered at deployment 36 . Collecting feedback on model usability given predictor missingness (e.g., Archer et al. 46 ) would aid future implementation. Role of Overall Clinical Prediction Models in PSCI Prognoses Clinical prediction models using acute cognitive information may offer more meaningful PSCI prognoses, as evidenced by better model performance compared to published prediction models of post-stroke cognition. Though PSCI rates are highest during acute stroke, early PSCI may be reversible 47 and information about likely 6-month outcomes is valuable to stroke patients 48 . Qualitative research suggests that focusing solely on cognitive decline as a possible PSCI outcome (e.g., Hbid et al. 9 ) may cause undue concern or at best be irrelevant 49 . Stroke survivors and families commonly report wanting personalized information about managing cognitive changes that are personalised to their circumstances 48 . Our models are an essential first step to providing person-specific cognitive trajectories. Strengths and Limitations A notable strength is using a stroke-specific PSCI outcome measure rather than an overall cognitive decline or dementia score. The OCS’ minimisation of confounds, brief administration time, and information on cognitive domains compromised in stroke (Language, Memory, Attention, Numeracy, Executive Function and Praxis) make it a credible candidate for PSCI model development. Though our overall PSCI performs well and is an improvement on cognitive decline-focused models, domain-specific models may provide further personalised detail still. Low prevalence 50 of certain domain-specific outcomes (numeracy, executive function, and praxis) restricted development of domain-specific models. Whilst highly common acutely, these impairments are less prevalent at the chronic stage and therefore require substantial sample sizes for sufficient development. Future domain-specific models could also consider how combinations of specific cognitive impairments influence performance. Specific impairment combinations (e.g., language and executive function) could however affect outcomes differently, given differential correlations between cognitive domains 13 . Developing within-domain models (e.g., sentence reading model vs language impairment model) may also be helpful, given varying recovery within domains 4 , 13 . Furthermore, predictor selection should be carefully considered. Baseline cognition best explains long-term PSCI risk 21 with established predictors other than age explaining little variance 13 , 42 . Less frequently researched PSCI domains (e.g. numeracy, praxis) may particularly benefit from data-driven predictors. Finally, as typical for new prediction models, recalibration of our models is required. Conclusion We demonstrate that a domain-specific model including acute cognitive information improved prediction of 6-month PSCI, with initial external validation in a milder cohort. Our model development process allows for future inclusion of novel data-driven predictors. This prediction model of stroke-specific cognition has the potential to offer more meaningful PSCI prognoses, including domain-specific cognitive recovery, compared to existing models focused on domain-general decline. Data Availability Data is freely availble at DementiaPlatforms UK https://dementiasplatforms.uk/ https://dementiasplatforms.uk/ Footnotes Author Emails andrea.kusec{at}ndcn.ox.ac.uk k.snell{at}bham.ac.uk Update manuscript with greater external validation analyses References 1. ↵ Feigin VL , Stark BA , Johnson CO , et al. Global, regional, and national burden of stroke and its risk factors, 1990–2019: a systematic analysis for the Global Burden of Disease Study 2019 . Lancet Neurol. 2021 ; 20 ( 10 ): 795 – 820 . doi: 10.1016/S1474-4422(21)00252-0 OpenUrl CrossRef PubMed 2. ↵ Pendlebury ST , Rothwell PM . Prevalence, incidence, and factors associated with pre-stroke and post-stroke dementia: a systematic review and meta-analysis . Lancet Neurol . 2009 ; 8 ( 11 ): 1006 – 1018 . doi: 10.1016/S1474-4422(09)70236-4 OpenUrl CrossRef PubMed Web of Science 3. ↵ Sexton E , McLoughlin A , Williams DJ , et al. Systematic review and meta-analysis of the prevalence of cognitive impairment no dementia in the first year post-stroke . Eur Stroke J . 2019 ; 4 ( 2 ): 160 – 171 . doi: 10.1177/2396987318825484 OpenUrl CrossRef PubMed 4. ↵ Kusec A , Milosevich E , Williams OA , et al. Long-term psychological outcomes following stroke: the OX-CHRONIC study . BMC Neurol . 2023 ; 23 ( 1 ): 426 . doi: 10.1186/s12883-023-03463-5 OpenUrl CrossRef PubMed 5. ↵ Mahon S , Parmar P , Barker-Collo S , et al. Determinants, Prevalence, and Trajectory of Long-Term Post-Stroke Cognitive Impairment: Results from a 4-Year Follow-Up of the ARCOS-IV Study . Neuroepidemiology . 2017 ; 49 ( 3-4 ): 129 – 134 . doi: 10.1159/000484606 OpenUrl CrossRef PubMed 6. ↵ Cumming TB , Brodtmann A , Darby D , Bernhardt J . The importance of cognition to quality of life after stroke . J Psychosom Res . 2014 ; 77 ( 5 ): 374 – 379 . doi: 10.1016/j.jpsychores.2014.08.009 OpenUrl CrossRef PubMed 7. ↵ Stolwyk RJ , Mihaljcic T , Wong DK , Hernandez DR , Wolff B , Rogers JM . Post-stroke Cognition is Associated with Stroke Survivor Quality of Life and Caregiver Outcomes: A Systematic Review and Meta-analysis . Neuropsychol Rev. Published online March 11 , 2024 . doi: 10.1007/s11065-024-09635-5 OpenUrl CrossRef 8. ↵ Luengo-Fernandez R , Violato M , Candio P , Leal J . Economic burden of stroke across Europe: A population-based cost analysis . Eur Stroke J . 2020 ; 5 ( 1 ): 17 – 25 . doi: 10.1177/2396987319883160 OpenUrl CrossRef PubMed 9. ↵ Hbid Y , Fahey M , Wolfe CDA , Obaid M , Douiri A . Risk Prediction of Cognitive Decline after Stroke . J Stroke Cerebrovasc Dis . 2021 ; 30 ( 8 ): 105849 . doi: 10.1016/j.jstrokecerebrovasdis.2021.105849 OpenUrl CrossRef PubMed 10. ↵ Tang EYH , Robinson L , Stephan BCM . Risk Prediction Models for Post-Stroke Dementia Geriatrics . 2017 ; 2 ( 3 ): 19 . doi: 10.3390/geriatrics2030019 OpenUrl CrossRef 11. ↵ Demeyere N , Sun S , Milosevich E , Vancleef K . Post-stroke cognition with the Oxford Cognitive Screen vs Montreal Cognitive Assessment: a multi-site randomized controlled study (OCS-CARE) . AMRC Open Res . 2019 ; 1 : 12 . doi: 10.12688/amrcopenres.12882.1 OpenUrl CrossRef 12. ↵ Rost NS , Brodtmann A , Pase MP , et al. Post-stroke cognitive impairment and dementia . Circulation research . 2022 ; 130 ( 8 ): 1252 – 1271 . OpenUrl CrossRef PubMed 13. ↵ Milosevich ET , Moore MJ , Pendlebury ST , Demeyere N . Domain-specific cognitive impairment 6lJmonths after stroke: The value of early cognitive screening . Int J Stroke. Published online September 25 , 2023 : 17474930231205787 . doi: 10.1177/17474930231205787 OpenUrl CrossRef 14. ↵ Moore MJ , Vancleef K , Riddoch MJ , Gillebert CR , Demeyere N . Recovery of Visuospatial Neglect Subtypes and Relationship to Functional Outcome Six Months After Stroke . Neurorehabil Neural Repair . 2021 ; 35 ( 9 ): 823 – 835 . doi: 10.1177/15459683211032977 OpenUrl CrossRef PubMed 15. ↵ Fahey M , Rudd A , Béjot Y , Wolfe C , Douiri A . Development and validation of clinical prediction models for mortality, functional outcome and cognitive impairment after stroke: a study protocol . BMJ Open . 2017 ; 7 ( 8 ): e014607 . doi: 10.1136/bmjopen-2016-014607 OpenUrl Abstract / FREE Full Text 16. Lee M , Yeo NY , Ahn HJ , et al. Prediction of post-stroke cognitive impairment after acute ischemic stroke using machine learning . Alzheimers Res Ther . 2023 ; 15 ( 1 ): 147 . doi: 10.1186/s13195-023-01289-4 OpenUrl CrossRef PubMed 17. ↵ Li X , Chen Z , Jiao H , et al. Machine learning in the prediction of post-stroke cognitive impairment: a systematic review and meta-analysis . Front Neurol . 2023 ; 14 : 1211733 . doi: 10.3389/fneur.2023.1211733 OpenUrl CrossRef 18. ↵ Stephan BCM , Minett T , Muniz Terrera G , Matthews FE , Brayne C . Dementia prediction for people with stroke in populations: is mild cognitive impairment a useful concept? Age Ageing . 2015 ; 44 ( 1 ): 78 – 83 . doi: 10.1093/ageing/afu085 OpenUrl CrossRef PubMed 19. ↵ Tang EYH , Price CI , Robinson L , et al. Assessing the Predictive Validity of Simple Dementia Risk Models in Harmonized Stroke Cohorts . Stroke . 2020 ; 51 ( 7 ): 2095 – 2102 . doi: 10.1161/STROKEAHA.120.027473 OpenUrl CrossRef PubMed 20. ↵ Mole JA , Demeyere N . The relationship between early post-stroke cognition and longer term activities and participation: A systematic review . Neuropsychol Rehabil . 2020 ; 30 ( 2 ): 346 – 370 . doi: 10.1080/09602011.2018.1464934 OpenUrl CrossRef PubMed 21. ↵ Filler J , Georgakis MK , Dichgans M . Risk factors for cognitive impairment and dementia after stroke: a systematic review and meta-analysis . Lancet Healthy Longev . 2024 ; 5 ( 1 ): e31 – e44 . doi: 10.1016/S2666-7568(23)00217-9 OpenUrl CrossRef 22. ↵ Demeyere N . Acute post-stroke screening for a cognitive care pathway . Lancet Healthy Longev . 2024 ; 5 ( 1 ): e4 – e5 . doi: 10.1016/S2666-7568(23)00257-X OpenUrl CrossRef 23. ↵ Lindsay P , Furie KL , Davis SM , Donnan GA , Norrving B . World Stroke Organization Global Stroke Services Guidelines and Action Plan . Int J Stroke . 2014 ; 9 ( SA100 ): 4 – 13 . doi: 10.1111/ijs.12371 OpenUrl CrossRef PubMed 24. Quinn TJ , Richard E , Teuschl Y , et al. European Stroke Organisation and European Academy of Neurology joint guidelines on post-stroke cognitive impairment . Eur Stroke J . 2021 ; 6 ( 3 ):I-XXXVIII. doi: 10.1177/23969873211042192 OpenUrl CrossRef 25. ↵ 25. Contents. National Clinical Guideline for Stroke. Accessed February 9, 2024. https://www.strokeguideline.org/contents/ 26. ↵ Demeyere N , Riddoch MJ , Slavkova ED , et al. Domain-specific versus generalized cognitive screening in acute stroke . J Neurol . 2016 ; 263 ( 2 ): 306 – 315 . doi: 10.1007/s00415-015-7964-4 OpenUrl CrossRef PubMed 27. ↵ Bisogno AL , Franco Novelletto L , Zangrossi A , et al. The Oxford cognitive screen (OCS) as an acute predictor of long-term functional outcome in a prospective sample of stroke patients . Cortex . 2023 ; 166 : 33 – 42 . doi: 10.1016/j.cortex.2023.04.015 OpenUrl CrossRef PubMed 28. ↵ Demeyere N , Riddoch MJ , Slavkova ED , Bickerton WL , Humphreys GW . The Oxford Cognitive Screen (OCS): Validation of a stroke-specific short cognitive screening tool . Psychol Assess . 2015 ; 27 ( 3 ): 883 – 894 . doi: 10.1037/pas0000082 OpenUrl CrossRef PubMed 29. ↵ R Core Team. R Studio . Published online 2024 . 30. ↵ Frank E Harrell Jr . . rms: Regression Modeling Strategies. Published online September 8, 2009:6.8-1. doi: 10.32614/CRAN.package.rms OpenUrl CrossRef 31. ↵ Heymans M. psfmi: Prediction Model Pooling, Selection and Performance Evaluation Across Multiply Imputed Datasets . Published online May 16, 2019 :1.4.0. doi: 10.32614/CRAN.package.psfmi OpenUrl CrossRef 32. ↵ Buuren S van , Groothuis-Oudshoorn K. mice: Multivariate Imputation by Chained Equations in R . J Stat Softw . 2011 ; 45 : 1 – 67 . doi: 10.18637/jss.v045.i03 OpenUrl CrossRef 33. ↵ Ensor J . pmvalsampsize: Sample Size for External Validation of a Prediction Model . Published online November 16, 2023 :0.1.0. doi: 10.32614/CRAN.package.pmvalsampsize OpenUrl CrossRef 34. ↵ Riley RD , Snell KI , Ensor J , et al. Minimum sample size for developing a multivariable prediction model: PART II - binary and time-to-event outcomes . Stat Med . 2019 ; 38 ( 7 ): 1276 – 1296 . doi: 10.1002/sim.7992 OpenUrl CrossRef PubMed 35. ↵ Riley RD , Ensor J , Snell KIE , et al. Calculating the sample size required for developing a clinical prediction model . BMJ . 2020 ; 368 : m441 . doi: 10.1136/bmj.m441 OpenUrl FREE Full Text 36. ↵ Sisk R , Sperrin M , Peek N , van Smeden M , Martin GP . Imputation and missing indicators for handling missing data in the development and deployment of clinical prediction models: A simulation study . Stat Methods Med Res . 2023 ; 32 ( 8 ): 1461 – 1477 . doi: 10.1177/09622802231165001 OpenUrl CrossRef PubMed 37. ↵ Smith G . Step away from stepwise . J Big Data . 2018 ; 5 ( 1 ): 32 . doi: 10.1186/s40537-018-0143-6 OpenUrl CrossRef 38. ↵ Steyerberg EW , Vickers AJ , Cook NR , et al. Assessing the performance of prediction models: a framework for some traditional and novel measures . Epidemiol Camb Mass . 2010 ; 21 ( 1 ): 128 – 138 . doi: 10.1097/EDE.0b013e3181c30fb2 OpenUrl CrossRef PubMed Web of Science 39. ↵ Collins GS , Reitsma JB , Altman DG , Moons KGM . Transparent Reporting of a Multivariable Prediction Model for Individual Prognosis or Diagnosis (TRIPOD) . Circulation . 2015 ; 131 ( 2 ): 211 – 219 . doi: 10.1161/CIRCULATIONAHA.114.014508 OpenUrl Abstract / FREE Full Text 40. ↵ Riley RD , Debray TPA , Collins GS , et al. Minimum sample size for external validation of a clinical prediction model with a binary outcome . Stat Med . 2021 ; 40 ( 19 ): 4230 – 4251 . doi: 10.1002/sim.9025 OpenUrl CrossRef PubMed 41. ↵ Tang EY , Amiesimaka O , Harrison SL , et al. Longitudinal Effect of Stroke on Cognition: A Systematic Review . J Am Heart Assoc . 7 ( 2 ): e006443 . doi: 10.1161/JAHA.117.006443 OpenUrl Abstract / FREE Full Text 42. ↵ Predicting post-stroke cognitive impairment using electronic health record data - Jeffrey M Ashburner, Yuchiao Chang, Bianca Porneala, Sanjula D Singh, Nirupama Yechoor, Jonathan M Rosand, Daniel E Singer, Christopher D Anderson , Steven J Atlas, 2024. Accessed August 5, 2024 . https://journals.sagepub.com/doi/full/10.1177/17474930241246156 43. ↵ Yuan X , Zhang L , Sui R , Wang Z . A risk prediction model of post-stroke cognitive impairment based on magnetic resonance spectroscopy imaging . Neurol Res . 2021 ; 43 ( 8 ): 642 – 652 . doi: 10.1080/01616412.2021.1908659 OpenUrl CrossRef PubMed 44. ↵ Simpkins AN , Janowski M , Oz HS , et al. Biomarker Application for Precision Medicine in Stroke . Transl Stroke Res . 2020 ; 11 ( 4 ): 615 – 627 . doi: 10.1007/s12975-019-00762-3 OpenUrl CrossRef PubMed 45. ↵ Demeyere N , Moore MJ . Innovations and challenges in predicting cognitive trajectories after stroke . Brain Commun . 2024 ; 6 ( 5 ):fcae364. 46. ↵ Archer L , Snell KIE , Stynes S , et al. Development and External Validation of Individualized Prediction Models for Pain Intensity Outcomes in Patients With Neck Pain, Low Back Pain, or Both in Primary Care Settings . Phys Ther . 2023 ; 103 ( 11 ):pzad128. doi: 10.1093/ptj/pzad128 OpenUrl CrossRef PubMed 47. ↵ Rasquin SMC , Lodder J , Verhey FRJ . Predictors of reversible mild cognitive impairment after stroke: a 2-year follow-up study . J Neurol Sci . 2005 ; 229 – 230 :21-25. doi: 10.1016/j.jns.2004.11.015 OpenUrl CrossRef PubMed 48. ↵ Lin B lei, Mei Y xia, Wang W na, et al. Unmet care needs of community-dwelling stroke survivors: a systematic review of quantitative studies . BMJ Open . 2021 ; 11 ( 4 ): e045560 . doi: 10.1136/bmjopen-2020-045560 OpenUrl Abstract / FREE Full Text 49. ↵ Hobden G , Tang E , Demeyere N . A qualitative study investigating the views of stroke survivors and their family members on discussing post-stroke cognitive trajectories . Neuropsychol Rehabil . 2024 ; 34 ( 10 ). doi: 10.1080/09602011.2024.2314882 OpenUrl CrossRef 50. ↵ Sor R , Park YS , Boets P , Goethals PLM , Lek S . Effects of species prevalence on the performance of predictive models . Ecol Model . 2017 ; 354 : 11 – 19 . doi: 10.1016/j.ecolmodel.2017.03.006 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted March 07, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment Andrea Kusec , Kym IE Snell , Nele Demeyere medRxiv 2024.09.06.24313196; doi: https://doi.org/10.1101/2024.09.06.24313196 Share This Article: Copy Citation Tools Development and validation of a clinical prediction model of domain-specific post-stroke cognitive impairment Andrea Kusec , Kym IE Snell , Nele Demeyere medRxiv 2024.09.06.24313196; doi: https://doi.org/10.1101/2024.09.06.24313196 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neurology Subject Areas All Articles Addiction Medicine (573) Allergy and Immunology (865) Anesthesia (302) Cardiovascular Medicine (4453) Dentistry and Oral Medicine (444) Dermatology (383) Emergency Medicine (609) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1515) Epidemiology (15242) Forensic Medicine (30) Gastroenterology (1131) Genetic and Genomic Medicine (6615) Geriatric Medicine (669) Health Economics (1001) Health Informatics (4552) Health Policy (1372) Health Systems and Quality Improvement (1614) Hematology (543) HIV/AIDS (1270) Infectious Diseases (except HIV/AIDS) (15929) Intensive Care and Critical Care Medicine (1106) Medical Education (624) Medical Ethics (147) Nephrology (670) Neurology (6625) Nursing (346) Nutrition (999) Obstetrics and Gynecology (1148) Occupational and Environmental Health (957) Oncology (3344) Ophthalmology (979) Orthopedics (369) Otolaryngology (421) Pain Medicine (436) Palliative Medicine (130) Pathology (665) Pediatrics (1696) Pharmacology and Therapeutics (693) Primary Care Research (714) Psychiatry and Clinical Psychology (5461) Public and Global Health (9252) Radiology and Imaging (2207) Rehabilitation Medicine and Physical Therapy (1371) Respiratory Medicine (1197) Rheumatology (597) Sexual and Reproductive Health (715) Sports Medicine (530) Surgery (714) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a02c86f68edbbbf0',t:'MTc3OTk2MzYzMQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();