A multimodal human-computer interaction dataset for neurocognitive user state evaluation

A multimodal human-computer interaction dataset for neurocognitive user state evaluation | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results A multimodal human-computer interaction dataset for neurocognitive user state evaluation View ORCID Profile Sai Zhang , View ORCID Profile Xinyu Bai , Charles Hartley-O’Dwyer , Hugh Warren , View ORCID Profile Frederike Beyer , View ORCID Profile Valdas Noreika doi: https://doi.org/10.1101/2025.09.03.673947 Sai Zhang 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sai Zhang For correspondence: zcatherine98{at}gmail.com Xinyu Bai 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Xinyu Bai Charles Hartley-O’Dwyer 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site Hugh Warren 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site Frederike Beyer 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Frederike Beyer Valdas Noreika 1 Centre for Brain and Behaviour, Department of Psychology, School of Biological and Behavioural Sciences, Queen Mary University of London Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Valdas Noreika Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract We introduce the Simulated Environment for Neurocognitive State Evaluation (SENSE-42), a multimodal dataset collected during user interactions with desktop computers. It is designed for studying spontaneous fluctuations in the neurocognitive state related to the tonic alertness of computer users, with recordings from 42 participants over 2-hour sessions. Within a simulated desktop environment, participants performed real-world routine tasks, including application switching, file management, typing, and web browsing. High-resolution data were recorded across physiological (electroencephalography, electrocardiography, respiration) and subjective modalities of alertness. At five-minute intervals, alertness state was reported using seven questions, addressing sleepiness (Karolinska Sleepiness Scale), mental and temporal demand, perceived performance, effort and frustration (NASA Task Load Index), as well as attentiveness. Behavioural data included keyboard, mouse and webcam inputs. Demographic information for the experience, habits, and preferences of computer usage was collected. In addition, individual differences in sleep quality were evaluated using the Pittsburgh Sleep Quality Index and the Epworth Sleepiness Scale. The SENSE-42 dataset can contribute to future research in user state monitoring, behavioural analysis and physiological computing. Background & Summary Understanding and optimizing task performance and interaction quality in human-computer interaction (HCI) is essential for maintaining productivity, improving user experience and minimising error-related safety hazards 1 ; 2 , particularly when users are engaged in prolonged and cognitively demanding activities on the computer. The performance and engagement of computer users are governed by their underlying neurocognitive states, which encompass cognitive, affective, and physiological processes, including attention, sleepiness, mental workload, and emotional responses such as frustration and perceived effort 3 ; 4 ; 5 . Monitoring spontaneous fluctuations in the neurocognitive state related to the tonic alertness of computer users can reflect these internal states, supporting sustained attention and stable task engagement over time 6 ; 7 . Understanding and evaluating tonic alertness in structured, task-driven HCI contexts is therefore critical for reducing frustration, increasing engagement, and enhancing overall satisfaction, which forms the focus of this work. Various approaches have been developed to assess alertness, broadly categorized into subjective and objective measurements. Subjective assessments, such as the Karolinska Sleepiness Scale (KSS) 8 , the Epworth Sleepiness Scale (ESS) 5 , and the NASA Task Load Index (NASA-TLX) 4 , provide insight into perceived cognitive states but suffer from challenges in reliability and temporal resolution 9 . Objective methods, ranging from task-based performance metrics 10 to physiological measures such as electroencephalogram (EEG) and cardiac signals 11 ; 12 , offer greater temporal granularity but are often more intrusive or technically demanding. Subjective measures provide a common scale for rating perceived alertness, enabling comparability across individuals and contexts. Conversely, objective measures facilitate continuous, minimally disruptive monitoring, mitigating the inherent limitations of self-reported data such as recall bias 13 or momentary introspective inaccuracy 14 . Together, these approaches offer complementary strengths, enabling a more accurate and holistic modelling of internal user states. Prior studies in HCI have shown substantial progress in linking user alertness with behavioural cues derived from keyboard usage, mouse movement, and eye-tracking data 15 ; 16 ; 17 . Pimenta et al. investigated the monitoring of mental fatigue through keyboard and mouse interaction patterns, focusing on generic, task-independent metrics such as keydown time, error rate per key press, mouse acceleration, and double-click speed, collected during two sessions at the start and end of each day 15 . Yamada et al. examined the detection of mental fatigue using eye-tracking data before and after cognitively demanding tasks, reporting improved accuracy over earlier approaches 16 . Natnithikarat et al. proposed a drowsiness detection method that integrates keyboard, mouse, and eye-tracking dynamics in a standalone application to simulate a working routine in the accounting division and perform automated quality checks on transactions 17 . While these efforts have demonstrated promising associations, several limitations remain. Most existing datasets rely heavily on subjective self-report measures, lack multimodal integration, and are based on unscheduled or loosely structured tasks without strict timings or a fixed experimental schedule, limiting their intrinsic validity and reliability 18 ; 19 . Moreover, few studies have focused on the continuous measurement of spontaneous fluctuations in user states, with most studies comparing only discrete time points such as before and after tasks or morning and evening effects 20 . Furthermore, few studies incorporate synchronised physiological and behavioural data in real-world-like computer usage scenarios 21 . The absence of reliable physiological data source from EEG, ECG or respiration limits the ability to link internal physiological states with external observable user behaviours, hindering the development of more accurate models and predictions of neurocognitive states across broader applications. To address these gaps, we introduce a multimodal dataset named S imulated E nvironment for N eurocognitive S tate E valuation (SENSE-42) 22 , designed to study early alertness state fluctuations during realistic computer interaction in a desktop operating system environment. To accommodate user preferences while maintaining a unified experimental design, we developed precise replicas of two major operating system interfaces (macOS 23 and Windows 24 ) to facilitate the study of task-related behavioural responses. Both interface styles were presented to each participant during the session, allowing within-subject comparisons across familiar and unfamiliar environments. The dataset includes synchronised high-resolution neurocognitive responses in multiple modalities, including physiological, behavioural, and self-reported data, from 42 participants within a simulated desktop operating system. These responses were collected over continuous 2-hour sessions for each participant, providing rich measurements of spontaneous fluctuations in user states. Structured tasks were designed to emulate everyday computer use—file management, typing, browsing, and application switching—under tightly controlled conditions. Participants self-reported their alertness state at five-minute intervals using predefined scales, including the Karolinska Sleepiness Scale (KSS) 8 , NASA Task Load Index (NASA-TLX) 4 , and attentiveness ratings 25 . Physiological data were collected to profile internal user states from EEG, ECG and respiration signals, and behavioural data were recorded for keyboard and mouse dynamics. Additionally, trait-level measures such as the Epworth Sleepiness Scale (ESS) 5 and the Pittsburgh Sleep Quality Index (PSQI) 26 were collected to contextualize inter-individual variability in daytime sleepiness and nighttime sleep patterns. The SENSE-42 dataset is collected in a time-locked experimental setting to replicate realworld computer use scenarios. It provides a structured resource for examining tonic alertness dynamics during human-computer interaction and may be used to explore relationships between various multimodal indicators of the user state and their personal traits. The dataset includes components relevant to research in areas such as cognitive state monitoring, user behaviour analysis, and physiological computing 27 . Methods Participants A total of 42 participants were recruited in our study, consisting of 21 males and 21 females. The average age of participants is 26.8 ± 4.5 years, ranging from 18 to 40. Participants represented diverse ethnic backgrounds, education levels, and occupations. To preserve ecological validity, they were instructed to maintain their usual intake of substances that could influence alertness, such as caffeine or medication. The study was approved by the Queen Mary Ethics of Research Committee (Reference Number: PSY2025-32), and the procedures were designed in accordance with the Data Protection Privacy Notice of Queen Mary University of London 28 . Detailed explanations of the study and its procedures were provided to the participants, and informed consent for data publication with optional data-sharing preferences was obtained from all participants. Participants received £30 for taking part in the study. Study Design Data collection was organised from March 2025 to July 2025 in the Centre for Brain and Behaviour at Queen Mary University of London. The overall experiment design is shown in Fig. 1 (a) . The study followed a single-group observational design, with all participants completing an identical task sequence in a fixed order. Additionally, a within-subject design was applied to evaluate user interactions across two conventional interface layouts within the simulated operating system (see Fig. 1 (b) ). This dataset 22 includes all EEG, ECG, respiration, keyboard, and mouse data from the calibration stage to the end of the two-hour session. The experiment sessions were conducted individually, with participants selecting their preferred time slot from 11:00, 13:00, or 15:00. Participants’ gender and operating system layout preferences were balanced at a 50:50 ratio to reduce response biases, and only individuals proficient with English and desktop computers were included to ensure consistent task engagement. The experiment was conducted in a sound-insulated chamber, with the experimenter seated in a separate control room to minimise external influence. Environmental conditions were strictly controlled: ambient noise was maintained at approximately 52 dB, room temperature was kept at normal range ( ∼ 25°C), lighting was controlled at standard workplace levels ( ∼ 500 Lux), and CO 2 concentration was kept below 800 ppm. Download figure Open in new tab Figure 1: Experiment design. a) Top view of the experiment setup. Participants were asked to perform tasks on a computer following instructions at the top of the screen, with keyboard and mouse as the main input devices. EEG, ECG, respiration data, and webcam recordings were collected throughout the session. b) Illustration of software UI/UX design complying with two conventional operating system layouts. c) Details of experiment flow with screenshots. The duration of the experiment session is two hours. Experimental design The illustration of overall experimental design is shown in Fig. 1 (c) . The experiment began with an introduction and calibration phase in which participants were introduced to two conventional operating system layouts (toolbar items in leading position: macOS 23 ; toolbar items in trailing position: Windows 24 ) and their corresponding UI/UX elements. Typing speed was calibrated in the next block to ensure time balance between keyboard- and mouse-based tasks. The main experiment lasted approximately two hours and consisted of a fixed set of individual or chained tasks within a fully controlled and simulated software environment. Tasks included: (1) opening or switching applications and responding to mail notifications; (2) performing application-specific operations such as dragging/opening folders, shadow typing or side-by-side typing, emptying the trash folder, or browsing webpages; and (3) closing or minimising the current application. Participants were instructed to open designated applications either by clicking the corresponding icon on the taskbar located at the bottom of the screen or by clicking on the mail notifications appearing on the right side of the screen after a 5-second pause. Once an application was launched, participants performed application-specific tasks within a continuous workflow. Mouse navigation tasks were confined within the central area of the application main window, occupying half of its width and height. Dragging folder task was repeated 20 times and opening folder task was repeated 30 times, with stimuli location randomised in each iteration. Typing tasks used word stimuli generated from a vocabulary list sampled according to word occurrence frequencies via the Faker library 29 and were designed to last approximately one minute per block. Participants performed two types of typing tasks: (1) shadow typing, where the target text was displayed in a lighter colour and participants typed over it, gradually replacing the reference as they typed; and (2) side-by-side typing, where the reference text remained visible on the left side of the screen while participants entered their response on the right. For file management, participants selected a set of files and confirmed their deletion via a pop-up dialogue. Web browsing tasks involved entering a typical URL generated by the Faker library 29 and completing the subject questionnaires embedded in a webpage every five minutes to assess their current state across seven dimensions: sleepiness (KSS) 8 , mental demand, temporal demand, performance, effort, frustration (NASA-TLX) 4 , and attentiveness (assessed with momentary mind-wandering scales) 25 . Finally, participants were instructed to hide the application window by clicking the minimise or close button located on the top toolbar. The order of tasks within each block was fixed to maintain a balance between mouse- and keyboard-driven interactions, while the system layout was randomly assigned between the two with equal probability and remained consistent throughout the same block. The questionnaires were presented in a non-intrusive manner, designed to minimally disrupt task engagement. The timing of questionnaire presentation was linked to the preceding task duration and controlled to appear approximately every five minutes. A screen recording of the experiment is provided in Supplementary Video S1. Throughout the session, multimodal data were continuously recorded, including EEG, ECG, respiration, behavioural interactions via mouse & keyboard, and self-reported responses. Within the behavioural data, mouse and keyboard responses including time, duration and location of each key press were synchronised with the screen refresh at 144 Hz and recorded on each frame. The timing of all stimulus presentations is included in the dataset, with the start of each test block linked to EEG triggers. Setup of physiological equipment was completed concurrently with the administration of pre-study questionnaires. Experiment guidelines were offered at the start of the experiment, encouraging participants to work efficiently and minimise pause between blocks. Experimental setup The experiment software 30 was implemented using PsychoPy v2024.2.3 and run on a Dell Precision 3490 workstation (Windows 11 Pro 24H2 24 ) equipped with an RTX 500 Ada GPU. The display was presented on a 144 Hz Dell G3223Q monitor, operating at the resolution of 1920×1080 with frame-to-frame refresh stability maintained below 1.5 ms. Cursor speed was set to the system’s medium setting. Participants interacted with a Dell SK-8175 keyboard and a Logitech M-U0017 optical mouse on a mousepad. A Dell WB5023 webcam operating at 1280×720 was used for video recordings where participant consent was provided. EEG data were recorded using a BioSemi ActiveTwo system with 32 electrodes arranged according to the extended 10–20 international placement system. Data were sampled at 1024 Hz, with electrode impedance maintained below 5 k Ω. EEG, ECG and Respiration acquisition was controlled via ActiView v8.0 running on a macOS 10.11.6 workstation. Environmental conditions were monitored using a Newentor C1 air quality sensor. The monitor and input devices were wired into the sound-attenuated chamber from the control room. Data Records The dataset 22 (DOI: 10.7303/syn68713182) is publicly available in open access and organised into six top-level directories reflecting the different modalities collected: Questionnaires, Behavioural, ECG, EEG, Respiration, and Webcam. Each directory contains flattened data files labelled with anonymized participant IDs. ECG signals were extracted from the auxiliary channels of the EEG recordings and saved in 3-channel .fif format using the MNE-Python library 31 . Respiration waveforms were sampled at 32 Hz and stored in .wav format, processed via the RespInPeace package 32 . EEG recordings are shared in the original BioSemi Data Format (BDF) on Synapse and cleaned EEGLab Format (.set) on Figshare Plus (DOI: 10.25452/figshare.plus.29971873). Webcam data are available upon request on Figshare Plus (DOI: 10.25452/figshare.plus.29973202) and subject to acceptance of specific terms of use; 23 out of 42 participants consented to share the raw video recordings for public access with access controls and all participants agreed to share their data for authorised academic use. Table 1 summarized the released set of pre- and in-study behavioural questionnaires. For details on how these questions were presented to participants, refer to the archived pre-study questionnaire document and the video recording of the experiment available in the dataset files. Further documentation, including field notes, contact details and an overview, is provided on the Synapse project wiki page. View this table: View inline View popup Download powerpoint Table 1: Released behavioural data from pre-study and in-study questionnaires. Technical Validation Behavioural data Participants completed an average of 34 ± 5 task blocks, with each block lasting approximately 227 ± 57 seconds. Within these blocks, 16 ± 4 trials were arranged in leading system toolbar layout and 18 ± 3 trials were in trailing system toolbar layout. The mean time spent on each task was 11.72 ± 9.50 seconds for opening the application, 28.12 ± 8.76 seconds for dragging folders, 23.60 ± 3.75 seconds for opening folders, 54.21 ± 18.85 seconds for shadow typing, 59.84 ± 26.61 seconds for side-by-side typing, 33.85 ± 18.48 seconds for web browsing, 3.44 ± 2.38 seconds for emptying the trash bin and 2.24 ± 1.63 seconds for closing or minimising the application. To validate task engagement and layout-specific behaviour, we present mouse navigation heatmaps separately for the two system layout conventions in Fig. 2 (a) . Mouse locations are mainly distributed around the taskbar, toolbar and the centre of application window. Additionally, logarithm count of key-presses on the keyboard are visualised in Fig. 2 (b) . The frequent appearance of letter E, T, A, O and rare occurrence of letter Z, J, Q, X are aligned with the previous study on the letter frequencies in the English vocabulary 33 . These statistics support the validity and reliability of the recorded behavioural responses across participants. Download figure Open in new tab Figure 2: Behavioural data validation through keyboard and mouse interaction records. a) Normalised mouse location heatmap on the screen under leading and trailing system layouts. The central concentration in both heatmaps indicates frequent common task engagement, while peripheral activity reflects interactions with operating system components. b) Logarithm count of keyboard key presses, mapped onto a standard UK keyboard layout. Darker colours indicate more frequent hits, demonstrating consistent interaction patterns with frequently used function and character keys. Physiological data To validate the physiological data quality and consistency over time, we applied standardised preprocessing and analysis pipelines across EEG, ECG, and respiration signals. We checked critical quality indicators for each physiological sensor, including the resting-state EEG power spectral density (PSD), event-related potential (ERP) associated with the visual onset of folder stimuli, distribution of cardiac timing metrics and distribution of respiration timing metrics. EEG data were processed using the Automagic toolbox 34 in MATLAB R2023a 35 , which facilitated automatic detection and removal of muscle and eye movement artifacts, identification and rejection of noisy channels, followed by spherical spline interpolation for identified bad channels and rejection of noisy segments. After preprocessing, we further computed the power spectral density for Cz using the Welch method 36 . The absolute power of each electrode was transformed into the logarithmic scale (1 dB = 10 log( µV 2 )) to represent the relative signal strength. The averaged power spectrum at Cz channel for all participants is shown in Fig. 3 (a) , and the power spectral density plots for each participant are presented in Supplementary Material S2. The spectrum fits the criteria of good data quality with a smooth decline towards higher frequency and a slight increase around the alpha band power 37 . The averaged event-related potential time-locked to the presentation of the computer home screen at Fz channel is presented in Fig. 3 (b) . Extracted epochs were de-trended and baseline-corrected using the interval -100 ms to 0 ms. The averaged waveform shows a clear negative deflection (N100) peaking at approximately 120 ms, followed by a positive deflection (P200) around 180 ms after stimulus onset. These components are consistent with well-established visual ERP responses, where the N100 is associated with early sensory processing and the P200 with subsequent perceptual evaluation and attention allocation 37 . Standard deviations are marked in gray in both figures. Download figure Open in new tab Figure 3: Physiological data validation through EEG, ECG, and respiration signals. a) Power spectral density (PSD) of Cz channel within 0-20 Hz, averaged across participants with standard deviations shown in gray. b) Event-related Potential (ERP) time-locked to visual stimulus onset at Fz channel, averaged across participants with standard deviations in gray. The vertical dashed line at 0 s denotes the appearance of the visual cue (computer home screen). c) Cardiac timing metrics for R-R interval and heart rate variability derived from 3-lead ECG recordings. d) Respiration timing metrics with inhalation and exhalation patterns. For cardiac signals, we extracted events from a derived 3-lead ECG time series and selected lead II signals calculated from the difference between the left leg (LL) and right arm (RA) to be processed by the MNE-Python 31 . Only valid heartbeats within the physiological range (50–220 bpm) were retained. We calculated mean R–R intervals and heart rate variability (HRV) from Standard Deviation of N-N intervals (SDNN) 38 as indicators of autonomic activity (as in Fig. 3 (c) ). The observed ranges for R–R intervals and HRV are within established short-term normative values 39 , indicating typical cardiac function in the recorded data. Respiration signals were processed using the RespInPeace package 32 , which applied low-pass filtering for noise reduction, baseline estimation to account for temporal drifting, and keypoint identification for inhalation and exhalation. Based on the RespInPeace package, we derived breathing-related duration and variability metrics for comparison in Fig. 3 (d) . The results are consistent with expected physiological ranges for healthy adults 40 , indicating that the recorded breathing patterns were within normal limits. Self-reported data Participants self-reported their attentiveness, mental demand, temporal demand, performance, effort, frustration, and sleepiness during the study, as shown in Fig. 4 . Sleepiness was assessed on a 9-point scale using the Karolinska Sleepiness Scale (KSS) 8 , while the other six dimensions were rated on a 7-point scale adapted from the NASA Task Load Index (NASA-TLX) 4 and momentary mind-wandering assessments 25 . The self-assessment blocks were arranged every 299 ± 43 seconds, consistent with the intended five-minute schedule. Download figure Open in new tab Figure 4: Distribution of self-reported ratings for attentiveness, mental demand, temporal demand, performance, effort, frustration, and sleepiness across all trials. Ratings for all metrics except sleepiness were measured using a 7-point scale adapted from NASA-TLX 4 and momentary mind-wandering assessments 25 , while sleepiness was assessed using the 9-point Karolinska Sleepiness Scale (KSS) 8 . Multi-modality sensor synchronisation To validate the synchronisation of multi-modal sensors, the combination of events extracted from behavioural and physiological data were assessed for temporal accuracy and completeness. EEG event markers exhibited a low missing rate of 0.93% across 94,436 behavioural events. The timing precision of received EEG triggers was validated against corresponding behavioural events, with an offset of 0.09 ± 8.10 ms, equivalent to ± 1 frame on a 144 Hz display. Webcam recordings were recorded in separate threads for each experimental block and saved as discrete MP4 files, enabling block-wise synchronisation and streamlined data handling. Usage Notes The dataset is organised in a flattened structure to facilitate ease of download and processing for individual or combined modalities. Data preprocessing and analysis pipeline were implemented using MNE 31 , NumPy 41 , Pandas 42 , and SciPy 43 . The source code for the experimental software is available under the ‘experiment’ branch in the GitHub repository 30 , while the main branch 44 contains the analysis pipeline for data verification and further exploration. Designed around computer users, this dataset supports the investigation of tonic alertness dynamics during human-computer interaction. It enables the study for physiological fingerprints of behavioural performance and supports analysis of within- and between-subject factors in spontaneous user state fluctuations 1 ; 2 . Beyond user state analysis, the dataset allows for user behaviour analysis based on individual traits and graphical interface design, offering opportunities to explore the physiological and behavioural patterns related to the experience, habits, preferences of computer usage and cross-system design patterns. Furthermore, through the integration of physiological signals with interface interactions, this dataset may further support the development of physiological computing studies 27 , where signals from the brain and body are transformed into adaptive inputs to optimize or personalize human-computer interfaces. Code Availability The code used to process and validate the data is available at the repository HCI-SENSE-42 44 open sourced on GitHub. Author Contributions Sai Zhang designed and conducted this research project and drafted the manuscript. Xinyu Bai provided support for code implementation and technical validation. Charles Hartley-O’Dwyer assisted with participant recruitment and study support. Hugh Warren assisted with the pilot study, participant recruitment and study support. Dr. Frederike Beyer supervised the study. Dr. Valdas Noreika conceptualised and supervised the study. All authors reviewed and approved the final version of the manuscript. Competing Interests The authors declare no competing interests. Acknowledgements Sai Zhang was funded by the China Scholarship Council scholarship (CSC No. 202309210085). We thank Jingni Yan for providing template questions related to mind wandering and for sharing relevant research studies; Khadeejah Jahan for coordinating study support; and Chi-Hsu Wu, our lab manager, for maintaining the experimental room equipment and conditions. Funder Information Declared China Scholarship Council, https://ror.org/04atp4p48 , 202309210085 Footnotes ↵ * z.sai{at}qmul.ac.uk https://www.synapse.org/Synapse:syn68713182/ https://github.com/Catherine9811/HCI-SENSE-42/ References [1]. ↵ Luca Longo . Experienced mental workload, perception of usability, their interaction and impact on task performance . PloS one , 13 ( 8 ): e0199661 , 2018 . OpenUrl CrossRef PubMed [2]. ↵ Alan Dix . Human-computer interaction . In Encyclopedia of database systems , pages 1327 – 1331 . Springer , 2009 . [3]. ↵ Nico Romeijn , Roy JEM Raymann , Els Møst , Bart Te Lindert , Wisse P Van Der Meijden , Rolf Fronczek , German Gomez-Herrero , and Eus JW Van Someren . Sleep, vigilance, and thermosensitivity . Pflügers Archiv-European Journal of Physiology , 463 ( 1 ): 169 – 176 , 2012 . OpenUrl [4]. ↵ Sandra G Hart and Lowell E Staveland . Development of nasa-tlx (task load index): Results of empirical and theoretical research . In Advances in psychology , volume 52 , pages 139 – 183 . Elsevier , 1988 . OpenUrl [5]. ↵ Murray W Johns . A new method for measuring daytime sleepiness: the epworth sleepiness scale . sleep , 14 ( 6 ): 540 – 545 , 1991 . OpenUrl CrossRef PubMed Web of Science [6]. ↵ Barry S Oken , Martin C Salinsky , and SM2865224 Elsas. Vigilance, alertness, or sustained attention: physiological basis and measurement . Clinical neurophysiology , 117 ( 9 ): 1885 – 1901 , 2006 . OpenUrl CrossRef PubMed Web of Science [7]. ↵ Joseph M DeGutis and Thomas M Van Vleet . Tonic and phasic alertness training: a novel behavioral therapy to improve spatial and non-spatial attention in patients with hemispatial neglect . Frontiers in Human Neuroscience , 4 : 60 , 2010 . [8]. ↵ Azmeh Shahid , Kate Wilkinson , Shai Marcu , and Colin M Shapiro . Karolinska sleepiness scale (kss) . STOP, THAT and one hundred other sleep scales , pages 209 – 210 , 2012 . [9]. ↵ Jessica Louise Lee , Yewon Chung , Edward Waters , and Hima Vedam . The epworth sleepiness scale: Reliably unreliable in a sleep clinic population . Journal of Sleep Research , 29 ( 5 ): e13019 , 2020 . OpenUrl PubMed [10]. ↵ Hans PA Van Dongen and David F Dinges . Sleep, circadian rhythms, and psychomotor vigilance . Clinics in sports medicine , 24 ( 2 ): 237 – 249 , 2005 . OpenUrl CrossRef PubMed Web of Science [11]. ↵ Chris Berka , Daniel J Levendowski , Milenko M Cvetinovic , Miroslav M Petrovic , Gene Davis , Michelle N Lumicao , Vladimir T Zivkovic , Miodrag V Popovic , and Richard Olmstead . Realime analysis of eeg indexes of alertness, cognition, and memory acquired with a wireless eeg headset . International Journal of Human-Computer Interaction , 17 ( 2 ): 151 – 170 , 2004 . OpenUrl CrossRef [12]. ↵ Suganiya Murugan , Jerritta Selvaraj , and Arun Sahayadhas . Detection and analysis: Driver state with electrocardiogram (ecg) . Physical and engineering sciences in medicine , 43 ( 2 ): 525 – 537 , 2020 . OpenUrl [13]. ↵ Steven S Coughlin . Recall bias in epidemiologic studies . Journal of clinical epidemiology , 43 ( 1 ): 87 – 91 , 1990 . OpenUrl CrossRef PubMed Web of Science [14]. ↵ Xuan Zhou , Sally A Ferguson , Raymond W Matthews , Charli Sargent , David Darwent , David J Kennaway , and Gregory D Roach . Mismatch between subjective alertness and objective performance under sleep restriction is greatest during the biological night . Journal of sleep research , 21 ( 1 ): 40 – 49 , 2012 . OpenUrl PubMed [15]. ↵ André Pimenta , Davide Carneiro , Paulo Novais , and José Neves . Monitoring mental fatigue through the analysis of keyboard and mouse interaction patterns . In International Conference on Hybrid Artificial Intelligence Systems , pages 222 – 231 . Springer , 2013 . [16]. ↵ Yasunori Yamada and Masatomo Kobayashi . Detecting mental fatigue from eye-tracking data gathered while watching video: Evaluation in younger and older adults . Artificial intelligence in medicine , 91 : 39 – 48 , 2018 . OpenUrl PubMed [17]. ↵ Sanurak Natnithikarat , Sirakorn Lamyai , Pitshaporn Leelaarporn , Narin Kunaseth , Phairot Autthasan , Thayakorn Wisutthisen , and Theerawit Wilaiprasitporn . Drowsiness detection for office-based workload with mouse and keyboard data . In 2019 12th Biomedical Engineering International Conference (BMEiCON) , pages 1 – 4 . IEEE , 2019 . [18]. ↵ Mindaugas Ulinskas , Robertas Damaševičius , Rytis Maskeliūnas , and Marcin Wózniak . Recognition of human daytime fatigue using keystroke data . Procedia computer science , 130 : 947 – 952 , 2018 . OpenUrl [19]. ↵ Vijay Prakash Sharma , Jitendra Singh Yadav , and Vivek Sharma . Deep convolutional network based real time fatigue detection and drowsiness alertness system . International Journal of Electrical and Computer Engineering (IJECE) , 12 ( 5 ): 5493 – 5500 , 2022 . OpenUrl [20]. ↵ Alyssa A Gamaldo and Jason C Allaire . Daily fluctuations in everyday cognition: is it meaningful? Journal of Aging and Health , 28 ( 5 ): 834 – 849 , 2016 . OpenUrl CrossRef PubMed [21]. ↵ Thomas Kosch , Jakob Karolus , Johannes Zagermann , Harald Reiterer , Albrecht Schmidt , and Pawel W Wózniak . A survey on measuring cognitive workload in human-computer interaction . ACM Computing Surveys , 55 ( 13s ): 1 – 39 , 2023 . OpenUrl CrossRef [22]. ↵ Sai Zhang , Xinyu Bai , Frederike Beyer , and Valdas Noreika . SENSE-42: A multimodal hci dataset from a simulated environment for neurocognitive state evaluation , 2025 . [23]. ↵ Apple Inc . macOS Sequoia 15 . https://www.apple.com/macos/ , 2024 . macOS Sequoia released on 16 September 2024. [24]. ↵ Microsoft . Windows operating system . https://www.microsoft.com/windows , 2024 . Windows 11 Pro 24H2 released on 15 June 2024. [25]. ↵ Issaku Kawashima , Tomoko Hinuma , and Saori C Tanaka . Ecological momentary assessment of mind-wandering: Meta-analysis and systematic review . Scientific Reports , 13 ( 1 ): 2873 , 2023 . OpenUrl PubMed [26]. ↵ Daniel J Buysse , Charles F Reynolds III . , Timothy H Monk , Susan R Berman , and David J Kupfer . The pittsburgh sleep quality index: a new instrument for psychiatric practice and research . Psychiatry research , 28 ( 2 ): 193 – 213 , 1989 . OpenUrl CrossRef PubMed Web of Science [27]. ↵ H. Silva , A. H. Holzinger , S. Fairclough , and D. Majoe . Physiological Computing Systems . Springer , 2014 . [28]. ↵ Queen Mary University of London . Data protection privacy notice for research participants . https://www.qmul.ac.uk/governance-and-legal-services/media/arcs/policyzone/Privacy-Notice-for-Research-Participants.pdf , 2025 . [29]. ↵ Gerome Fournier and Contributors . Faker: Python package for generating fake data . https://pypi.org/project/Faker/0.7.4/ , 2016 . [30]. ↵ Xinyu Bai and Sai Zhang . HCI-SENSE-42: Experimental software implementation . https://github.com/Catherine9811/HCI-SENSE-42/tree/experiment , 2025 . Implemented using PsychoPy v2024.2.3. [31]. ↵ Alexandre Gramfort , Martin Luessi , Eric Larson , Denis A. Engemann , Daniel Strohmeier , Christian Brodbeck , Roman Goj , Mainak Jas , Teon Brooks , Lauri Parkkonen , and Matti S. Hämäläinen . MEG and EEG data analysis with MNE-Python . Frontiers in Neuroscience , 7 ( 267 ): 1 – 13 , 2013 . OpenUrl PubMed [32]. ↵ Marcin Wlodarczak . Respinpeace: toolkit for processing respiratory belt data . In Fonetik 2019 , Stockholm, Sweden , June 10–12 , 2019, pages 115 – 118 , 2019 . OpenUrl [33]. ↵ Michael N Jones and Douglas JK Mewhort . Case-sensitive letter and bigram frequency counts from large-scale english corpora . Behavior research methods, instruments, & computers , 36 ( 3 ): 388 – 396 , 2004 . OpenUrl CrossRef PubMed Web of Science [34]. ↵ Andreas Pedroni , Amirreza Bahreini , and Nicolas Langer . Automagic: Standardized preprocessing of big eeg data . NeuroImage , 200 : 460 – 473 , 2019 . OpenUrl CrossRef PubMed [35]. ↵ MATLAB. version 7.10.0 (R2010a) . The MathWorks Inc ., Natick, Massachusetts , 2010 . [36]. ↵ Peter Welch . The use of fast fourier transform for the estimation of power spectra: a method based on time averaging over short, modified periodograms . IEEE Transactions on audio and electroacoustics , 15 ( 2 ): 70 – 73 , 1967 . OpenUrl CrossRef [37]. ↵ Steven J Luck . An introduction to the event-related potential technique . MIT press , 2014 . [38]. ↵ Robert E Kleiger , Phyllis K Stein , and J Thomas Bigger Jr . . Heart rate variability: measurement and clinical utility . Annals of noninvasive electrocardiology , 10 ( 1 ): 88 – 101 , 2005 . OpenUrl CrossRef PubMed Web of Science [39]. ↵ Fred Shaffer and Jay P Ginsberg . An overview of heart rate variability metrics and norms . Frontiers in public health , 5 : 258 , 2017 . [40]. ↵ John Burnard West . Respiratory physiology: the essentials. Lippincott Williams & Wilkins , 2012 . [41]. ↵ Charles R. Harris , K. Jarrod Millman , Sté fan J van der Walt , Ralf Gommers , Pauli Virtanen , David Cournapeau , Eric Wieser , Julian Taylor , Sebastian Berg , Nathaniel J. Smith , Robert Kern , Matti Picus , Stephan Hoyer , Marten H. van Kerkwijk , Matthew Brett , Allan Haldane , Jaime Fernández del Río , Mark Wiebe , Pearu Peterson , Pierre Gérard-Marchant , Kevin Sheppard , Tyler Reddy , Warren Weckesser , Hameer Abbasi , Christoph Gohlke , and Travis E. Oliphant . Array programming with NumPy . Nature , 585 : 357 – 362 , 2020 . OpenUrl CrossRef PubMed [42]. ↵ The pandas development team. pandas-dev/pandas: Pandas , February 2020 . [43]. ↵ Pauli Virtanen , Ralf Gommers , Travis E. Oliphant , Matt Haberland , Tyler Reddy , David Cournapeau , Evgeni Burovski , Pearu Peterson , Warren Weckesser , Jonathan Bright , Sté fan J. van der Walt , Matthew Brett , Joshua Wilson , K. Jarrod Millman , Nikolay Mayorov , Andrew R. J. Nelson , Eric Jones , Robert Kern , Eric Larson , C J Carey , İlhan Polat , Yu Feng , Eric W. Moore , Jake VanderPlas , Denis Laxalde , Josef Perktold , Robert Cimrman , Ian Henriksen , E. A. Quintero , Charles R. Harris , Anne M. Archibald , Antônio H. Ribeiro , Fabian Pedregosa , Paul van Mulbregt , and SciPy 1.0 Contributors . SciPy 1.0: Fundamental Algorithms for Scientific Computing in Python . Nature Methods , 17 : 261 – 272 , 2020 . OpenUrl PubMed [44]. ↵ Sai Zhang and Xinyu Bai . HCI-SENSE-42: Data processing and validation codebase . https://github.com/Catherine9811/HCI-SENSE-42 , 2025 . Tested with Python 3.9 and RStudio 2024.04.2. View the discussion thread. Back to top Previous Next Posted September 08, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following A multimodal human-computer interaction dataset for neurocognitive user state evaluation Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share A multimodal human-computer interaction dataset for neurocognitive user state evaluation Sai Zhang , Xinyu Bai , Charles Hartley-O’Dwyer , Hugh Warren , Frederike Beyer , Valdas Noreika bioRxiv 2025.09.03.673947; doi: https://doi.org/10.1101/2025.09.03.673947 Share This Article: Copy Citation Tools A multimodal human-computer interaction dataset for neurocognitive user state evaluation Sai Zhang , Xinyu Bai , Charles Hartley-O’Dwyer , Hugh Warren , Frederike Beyer , Valdas Noreika bioRxiv 2025.09.03.673947; doi: https://doi.org/10.1101/2025.09.03.673947 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Neuroscience Subject Areas All Articles Animal Behavior and Cognition (7618) Biochemistry (17636) Bioengineering (13859) Bioinformatics (41847) Biophysics (21401) Cancer Biology (18535) Cell Biology (25423) Clinical Trials (138) Developmental Biology (13353) Ecology (19860) Epidemiology (2067) Evolutionary Biology (24287) Genetics (15582) Genomics (22463) Immunology (17701) Microbiology (40300) Molecular Biology (17141) Neuroscience (88432) Paleontology (666) Pathology (2825) Pharmacology and Toxicology (4813) Physiology (7633) Plant Biology (15107) Scientific Communication and Education (2042) Synthetic Biology (4285) Systems Biology (9808) Zoology (2267)