iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour

iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour View ORCID Profile Elyse Letts , View ORCID Profile Damien Masson , View ORCID Profile Joyce Obeid doi: https://doi.org/10.1101/2025.04.17.25326032 Elyse Letts 1 Child Health & Exercise Medicine Program, Department of Pediatrics, McMaster University , 1280 Main St W, Hamilton, ON, L8S 4L8, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Elyse Letts Damien Masson 2 Université de Montréal , 2900 Edouard Montpetit Blvd, Montreal, QC, H3T 1J4, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Damien Masson Joyce Obeid 1 Child Health & Exercise Medicine Program, Department of Pediatrics, McMaster University , 1280 Main St W, Hamilton, ON, L8S 4L8, Canada 3 Department of Kinesiology, McMaster University , 1280 Main St W, Hamilton, ON, L8S 4L8, Canada Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Joyce Obeid For correspondence: obeidj{at}mcmaster.ca Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract We present iPlayer, a video annotation tool for direct observation of human activity. In contrast to other tools generally designed for continuous or intermittent annotation, iPlayer is optimized for annotations of specific time windows. The tool is cross-platform (MacOS, Windows, and Linux), open-source, locally-run, and configurable to support different labels and time window sizes. Users can annotate multiple activities per window and the video can be optionally looped over the window, reducing annotation error particularly for small window sizes. We developed and used the tool to annotate toddler activity and position in 1-second windows to be synchronized with raw accelerometer data. We also provide examples of how iPlayer could be used with other annotation frameworks. While the tool was optimized for annotations of human activity, we believe it may be of use to any researchers looking to precisely annotate segments of video recordings. Introduction When developing or validating accelerometer methods for human behaviour, having an accurate ground truth is essential ( Clevenger et al., 2022 ; Lyden et al., 2014 ). A gold-standard for ground truth in human movement analysis is direct observation ( Hukka, 2020 ; Letts et al., 2024 ; Lyden et al., 2014 ). A 2024 review found that almost half of the studies that validate methods against direct observation use a video recording ( Letts et al., 2024 ). These video recordings of activities of interest or free-living sessions are then annotated continuously or in fixed windows ( Letts et al., 2024 ). These video annotations can then be linked by time to the accelerometer data and used to train machine learning models or validate existing methods. Key in this process is how the video annotation is completed. Many tools exist to facilitate this video annotation ranging from manual to fully-automated methods. Automated methods, which generally use video recognition models, can speed up the annotation task by reducing time and personnel requirements, however, they introduce an extra layer of potential bias by using previously developed recognition models ( Demrozi et al., 2023 ). Manual methods, considered the gold-standard, can be very time consuming, so having a tool that facilitates the process is important ( Friard & Gamba, 2016 ). There are a number of existing video annotation tools that are commonly used. Some tools have a high financial cost (e.g., Noldus Observer XT ( Zimmerman et al., 2009 )) which may be prohibitive. Others are free, including BORIS ( Friard & Gamba, 2016 ), JWatcher ( Stankowich, 2008 ), and Datavyu ( Datavyu Team, 2014 ). These tools provide comprehensive annotation capabilities supporting many different types of annotations. The flipside of this comprehensive nature is that they may be overly complex for more straightforward video annotation tasks. For example, JWatcher requires at least 3 unique file types to set up the annotations and can generate more than 10 types of output files. Additionally, none of these free tools have the ability to loop over a specific time frame. When very high-resolution annotations are required (e.g., 1 second windows), it can be difficult to annotate on a non-stopping video. Annotators may miss the window, having to constantly go back, or be unclear where a window starts and ends. This could increase the error in the annotations intended to be ground truth, resulting in poorer outcomes if the annotations are used to develop new or validate existing methods. For example, if annotation windows are synchronized with raw accelerometer data windows, having unclear window edges may reduce accuracy of annotations and thus reduce performance of a newly trained machine learning model ( Peters et al., 2024 ). This brief technical report presents a free, open-source, locally-run tool, named iPlayer. iPlayer is optimized for time-windowed annotations: the video is segmented based on the time window; each segment is looped to help annotators correctly identify the movement happening in the particular segment; and the tool is configurable to support many annotation frameworks. While video annotations tools are not novel, we feel that the tool may be beneficial to other researchers completing similar video annotation tasks and thus are sharing the details of the tool and providing open-source access. Description of the tool iPlayer has three main views, as shown in Figure 1 : (A) a video player showing the current video segment being annotated with a few configuration options; (B) a timeline indicating which windows have been annotated and allowing to jump to specific windows; (C) an annotation panel listing all the possible options, as configured by the researchers. Download figure Open in new tab Figure 1. Screenshot of iPlayer in use with three main views highlighted: (A) the video player and its video options (speed, mute, loop, pause/play); (B) the timeline showing windows that are fully-(dark blue), partially-(light blue), or not-(white) annotated with yellow indicating windows with comments and the red showing current position; and (C) the annotation panel showing the annotation framework and Notes section. The video player area has controls to speed up or slow down the video (playback speed), mute the video, loop the video over the window, and refresh the video player in cases of video freezing. It also shows the annotations (once selected) in yellow for ease of review. The timeline uses colours to indicate completeness: dark blue indicates that both annotations are complete (e.g., both activity and position), light blue indicates that one annotation is complete (e.g., position but not activity), white indicates that no annotation has been entered, yellow indicates a comment in the Notes section, and the red indicator shows current position in the video. Users can move between time windows by clicking on the timeline, using arrow keys, or using the enter key to advance. The annotation panel shows the user defined annotation framework. Values can be entered using key shortcuts or by selecting the radio buttons. Notes are entered by clicking into the Notes box which temporarily disables the key shortcuts to allow free typing. Key shortcuts are re-enabled by clicking outside of the Notes box. Video files are loaded first (File>Load video), followed by either generating a new output .csv file (File>Save as) or by loading a previously started .csv (File>Load csv). The output .csv has the following headings: (1) Index: sequentially numbered windows; (2) Timestamp: the time in seconds from the start of the video for the start of that window; (3) User defined annotation 1 (e.g., Activity in example in Figure 1 ): the selection from the first annotation; (4) User defined annotation 2 (e.g., Position in example in Figure 1 ): the selection from the second annotation; (5) Notes: the string entered in the notes section. Tool configuration Labels, shortcuts, and the duration of the time window can be configured in an XML-file placed at the root of the application (the same folder as the application). By modifying this file, researchers can customize the interface to support different annotation tasks. For example, this file replicates the configuration we have been using to annotate toddler data over 1-second time windows using a modified version of the Children’s Activity Rating Scale ( Puhl et al., 2013 ) and adding position labels. The position labels are “multichoice” meaning that more than one label can be entered for a given window. This allows users to capture transitions in position within the time window (e.g., the participant sits up from the floor). Download figure Open in new tab Download figure Open in new tab Implementation iPlayer was developed in Kotlin ( Kotlin Programming Language , 2011 ), using JavaFX ( JavaFX , 2008 ) for the graphical user interface. The tool is cross-platform (MacOS, Windows, Linux) and open source. The code and downloadable software application can be found here: https://github.com/LettsE/iPlayer . Preliminary Evaluation We developed iPlayer as a more efficient alternative to Datavyu for our needs to annotate toddler physical activity, sedentary time, and position in one second windows (received ethics clearance from the Hamilton integrated Research Ethics Board; HiREB #3674). The tool was designed through multiple iterations based on the feedback received from annotators. In total, 19 annotators used it (on both Windows and MacOS) to annotate a total of 218 videos. The videos were recordings of toddler play sessions and ranged from 43 to 100 minutes in length. Annotators had to review and annotate each second of these videos and it took about six hours on average to complete a video. This allowed us to test the robustness of the tool on long videos that were annotated over multiple sessions. While we cannot guarantee the tool works for all users or use cases, we note that all annotators successfully used the tool with no major issue. One annotator commented on how much of an improvement the tool was, specifically the looping feature, over Datavyu for our specific use case. To further demonstrate the flexibility of iPlayer, we have selected four representative annotation frameworks from the Letts et al. (2024) review and generated their XML configuration files. The four annotation frameworks are: (1) the System for Observing Fitness Instruction Time (SOFIT) as used by Capio et al. (2010) in 15s windows; (2) steps or pedal revolution counts as described by Gatti et al. (2016) in 60s windows; (3) a custom activity list combined with MET classification as used by Hedayatrad et al. (2020) in 5s windows; and (4) a custom activity list designed to match the ActivPAL output including an “other” category as used by Alghaeed et al. (2013) in 1s windows. For (1) the SOFIT, the XML configuration file could be as follows: Download figure Open in new tab For (2) the step/pedal revolution counting, the configuration file could be as follows with the number of steps/revolutions entered in the notes section: Download figure Open in new tab For (3) the custom activity list combined with METs, the configuration file could be: Download figure Open in new tab Finally, (4) the custom activity list designed to match the ActivPAL output could have a configuration file as follows, using the Notes section to describe what the “other” is: Download figure Open in new tab Download figure Open in new tab Each of the above sample configuration files can be further customized by users. These examples demonstrate the flexibility of iPlayer to adapt to many annotation frameworks and window sizes. Discussion This paper presents iPlayer, a simple, open-source tool to annotate video recordings in windowed segments. It is designed to help researchers who need to label ground truth activity. It allows for window size and annotations to be customized, with multiple annotations per window as well as a notes feature. It outputs a .csv file which can be saved and reloaded into the tool to revise or resume partially completed annotations. The notes feature can be used to synchronize annotations to other data based on an occurrence in the video (e.g., clapping, clock shown in video). It may not, however, be suitable for all direct observation needs. iPlayer does not provide a method for live annotation nor for non-windowed (continuous timestamped) annotation. Additionally, users with more complex annotation needs or who need more customization than is currently available may prefer using one of the other existing tools such as BORIS or Datavyu. Conclusion This paper presents iPlayer, an open-source tool to simplify windowed video annotation of human activity and describes an example use cases. It is a video player combined with an annotation tool in which labels and window length can be customized. Its ability to loop the video over a selected window provides a benefit over existing open-source tools and can help improve the accuracy of windowed annotations. We hope that this tool may be of use to other researchers looking to manually annotate video recordings. Data Availability The code and downloadable software application can be found here: https://github.com/LettsE/iPlayer https://github.com/LettsE/iPlayer References ↵ Alghaeed , Z. , Reilly , J. J. , Chastin , S. F. M. , Martin , A. , Davies , G. , & Paton , J. Y. ( 2013 ). The Influence of Minimum Sitting Period of the ActivPAL TM on the Measurement of Breaks in Sitting in Young Children . PLoS One , 8 ( 8 ). doi: 10.1371/journal.pone.0071854 OpenUrl CrossRef ↵ Capio , C. M. , Sit , C. H. , & Abernethy , B. ( 2010 ). Physical activity measurement using MTI (actigraph) among children with cerebral palsy . Archives of Physical Medicine and Rehabilitation , 91 ( 8 ), 1283 – 1290 . doi: 10.1016/j.apmr.2010.04.026 OpenUrl CrossRef PubMed ↵ Clevenger , K. A. , Mackintosh , K. A. , McNarry , M. A. , Pfeiffer , K. A. , Nelson , M. B. , Bock , J. M. , Imboden , M. T. , Kaminsky , L. A. , & Montoye , A. H. K. ( 2022 ). A consensus method for estimating physical activity levels in adults using accelerometry . Journal of Sports Sciences , 40 ( 21 ), 2393 – 2400 . doi: 10.1080/02640414.2022.2159117 OpenUrl CrossRef PubMed ↵ Datavyu Team . ( 2014 ). Datavyu: A Video Coding Tool . [Computer software]. Databrary Project, New York University . http://datavyu.org ↵ Demrozi , F. , Turetta , C. , Machot , F. A. , Pravadelli , G. , & Kindt , P. H. ( 2023 ). A Comprehensive Review of Automated Data Annotation Techniques in Human Activity Recognition (No . arXiv: 2307.05988 ). arXiv. doi: 10.48550/arXiv.2307.05988 OpenUrl CrossRef ↵ Friard , O. , & Gamba , M. ( 2016 ). BORIS: A free, versatile open-source event-logging software for video/audio coding and live observations . Methods in Ecology and Evolution , 7 ( 11 ), 1325 – 1330 . doi: 10.1111/2041-210X.12584 OpenUrl CrossRef ↵ Gatti , A. A. , Stratford , P. W. , Brenneman , E. C. , & Maly , M. R. ( 2016 ). GT3X+ accelerometer placement affects the reliability of step-counts measured during running and pedal-revolution counts measured during bicycling . Journal of Sports Sciences , 34 ( 12 ), 1168 – 1175 . doi: 10.1080/02640414.2015.1096018 OpenUrl CrossRef PubMed ↵ Hedayatrad , L. , Stewart , T. , & Duncan , S. ( 2020 ). Concurrent Validity of ActiGraph GT3X+ and Axivity AX3 Accelerometers for Estimating Physical Activity and Sedentary Behavior . Journal for the Measurement of Physical Behaviour , 4 ( 1 ), 1 – 8 . doi: 10.1123/jmpb.2019-0075 OpenUrl CrossRef ↵ Hukka , M. ( 2020 ). Choice of criterion measure matters for validating estimates of youth sedentary behavior: Direct observation versus indirect calorimetry versus both . Masters Theses . https://trace.tennessee.edu/utk_gradthes/5630 ↵ JavaFX . ( 2008 ). [Computer software]. https://openjfx.io/ ↵ Kotlin Programming Language . ( 2011 ). [Computer software]. https://kotlinlang.org/ ↵ Letts , E. , Jakubowski , J. S. , King-Dowling , S. , Clevenger , K. A. , Kobsar , D. , & Obeid , J. ( 2024 ). Accelerometer techniques for capturing human movement validated against direct observation: A scoping review . Physiological Measurement , 45 ( 7 ). doi: 10.1088/1361-6579/ad45aa OpenUrl CrossRef ↵ Lyden , K. , Petruski , N. , Mix , S. , Staudenmayer , J. , & Freedson , P. ( 2014 ). Direct Observation is a Valid Criterion for Estimating Physical Activity and Sedentary Behavior . Journal of Physical Activity and Health , 11 ( 4 ), 860 – 863 . doi: 10.1123/JPAH.2012-0290 OpenUrl CrossRef ↵ Peters , H. , Hashemi , A. , & Rae , J. ( 2024 ). Generalizable Error Modeling for Human Data Annotation: Evidence From an Industry-Scale Search Data Annotation Program . J. Data and Information Quality , 16 ( 3 ), 17 :1-17:15. doi: 10.1145/3688394 OpenUrl CrossRef ↵ Puhl , J. , Greaves , K. , Hoyt , M. , & Baranowski , T. ( 2013 ). Children’s Activity Rating Scale (CARS): Description and Calibration . Research Quarterly for Exercise and Sport , 61 ( 1 ), 26 – 36 . doi: 10.1080/02701367.1990.10607475 OpenUrl CrossRef ↵ Stankowich , T. ( 2008 ). Quantifying Behavior the JWatcher Way. Daniel T. Blumstein and Janice C. Daniel . Integrative and Comparative Biology , 48 ( 3 ), 437 – 439 . doi: 10.1093/icb/icn005 OpenUrl CrossRef ↵ Zimmerman , P. H. , Bolhuis , J. E. , Willemsen , A. , Meyer , E. S. , & Noldus , L. P. J. J. ( 2009 ). The Observer XT: A tool for the integration and synchronization of multimodal signals . Behavior Research Methods , 41 ( 3 ), 731 – 735 . doi: 10.3758/BRM.41.3.731 OpenUrl CrossRef PubMed Web of Science View the discussion thread. Back to top Previous Next Posted April 20, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour Elyse Letts , Damien Masson , Joyce Obeid medRxiv 2025.04.17.25326032; doi: https://doi.org/10.1101/2025.04.17.25326032 Share This Article: Copy Citation Tools iPlayer: an open-source tool for time-windowed video annotation of human physical activity and behaviour Elyse Letts , Damien Masson , Joyce Obeid medRxiv 2025.04.17.25326032; doi: https://doi.org/10.1101/2025.04.17.25326032 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Epidemiology Subject Areas All Articles Addiction Medicine (567) Allergy and Immunology (863) Anesthesia (296) Cardiovascular Medicine (4409) Dentistry and Oral Medicine (443) Dermatology (380) Emergency Medicine (606) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1505) Epidemiology (15203) Forensic Medicine (30) Gastroenterology (1119) Genetic and Genomic Medicine (6570) Geriatric Medicine (666) Health Economics (994) Health Informatics (4510) Health Policy (1365) Health Systems and Quality Improvement (1608) Hematology (537) HIV/AIDS (1263) Infectious Diseases (except HIV/AIDS) (15902) Intensive Care and Critical Care Medicine (1103) Medical Education (620) Medical Ethics (144) Nephrology (665) Neurology (6572) Nursing (345) Nutrition (998) Obstetrics and Gynecology (1139) Occupational and Environmental Health (954) Oncology (3319) Ophthalmology (967) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (662) Pediatrics (1689) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5421) Public and Global Health (9205) Radiology and Imaging (2190) Rehabilitation Medicine and Physical Therapy (1367) Respiratory Medicine (1191) Rheumatology (593) Sexual and Reproductive Health (709) Sports Medicine (529) Surgery (709) Toxicology (99) Transplantation (288) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9fe738de4b04df88',t:'MTc3OTIzNjkxNA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();