Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research

Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research View ORCID Profile Kevin B. Johnson , View ORCID Profile Basam Alasaly , View ORCID Profile Kuk Jin Jang , Eric Eaton , View ORCID Profile Sriharsha Mopidevi , Ross Koppel doi: https://doi.org/10.1101/2025.05.18.25327837 Kevin B. Johnson 1 Division of Biomedical Informatics, Department of Biostatistics, Epidemiology, and Informatics, Perelman School of Medicine at the University of Pennsylvania , Philadelphia, USA 2 Department of Computer and Information Science, School of Engineering and Applied Science, University of Pennsylvania , Philadelphia, USA MD, MS, FACMI, FAMIA, FAAP, FIAHSI, FAIMBE Roles: David L. Cohen University Professor Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kevin B. Johnson For correspondence: Kevin.johnson1{at}pennmedicine.upenn.edu Basam Alasaly 3 Division of Biomedical Informatics, Department of Biostatistics, Epidemiology, and Informatics, Perelman School of Medicine, University of Pennsylvania , Philadelphia, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Basam Alasaly Kuk Jin Jang 4 Division of Biomedical Informatics, Department of Biostatistics, Epidemiology, and Informatics, Perelman School of Medicine at the University of Pennsylvania , Philadelphia, USA 5 Department of Computer and Information Science, School of Engineering and Applied Science, University of Pennsylvania , Philadelphia, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Kuk Jin Jang Eric Eaton 6 Department of Computer and Information Science, School of Engineering and Applied Science, University of Pennsylvania , Philadelphia, USA PhD Find this author on Google Scholar Find this author on PubMed Search for this author on this site Sriharsha Mopidevi 7 Division of Biomedical Informatics, Department of Biostatistics, Epidemiology, and Informatics, Perelman School of Medicine, University of Pennsylvania , Philadelphia, USA MS Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Sriharsha Mopidevi Ross Koppel 8 Division of Biomedical Informatics, Department of Biostatistics, Epidemiology, and Informatics, Perelman School of Medicine at the University of Pennsylvania , Philadelphia, USA 9 School of Arts and Sciences, University of Pennsylvania , Philadelphia, USA PhD, MA, FACMI, FIAHSI Roles: Professor of Sociology Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Objective To support ambulatory care innovation, we created Observer , a multimodal dataset comprising videotaped outpatient visits, electronic health record (EHR) data and structured surveys. This paper describes the data collection procedures and summarizes the clinical and contextual features of the dataset. Materials and Methods A multistakeholder steering group shaped recruitment strategies, survey design, and privacy-preserving design. Consented patients and primary care providers (PCPs) were recorded using room-view and egocentric cameras. EHR data, metadata and audit logs were also captured. A custom de-identification pipeline, combining transcript redaction, voice masking, and facial blurring, ensured video and EHR HIPAA compliance. Results We report on the first 100 visits in this continually growing dataset. Thirteen PCPs from four clinics participated. Recording the first 100 visits required approaching 210 patients, from which 129 consented (61%), with 29 patients missing their scheduled encounter after consenting. Visit lengths ranged from 5 to 100 minutes, covering preventive care to chronic disease management. Survey responses revealed high satisfaction: 4.24/5 (patients) and 3.94/5 (PCPs). Visit experience was unaffected by the presence of video recording technology. Discussion We demonstrate the feasibility of capturing rich, real-world primary care interactions using scalable, privacy-sensitive methods. Room layout and camera placement were key influences on recorded communication and are now added to the dataset. The Observer dataset enables future clinical AI research/development, communication studies, and informatics education among public and private user groups. Conclusion Observer is a new, shareable, real-world clinic encounter research and teaching resource with a representative sample of adult primary care data. Background The U.S. healthcare system—once a global exemplar—is now criticized for its high costs and relatively poor health outcomes. 1 While biomedical researchers and industry leaders have proposed incremental improvements to existing technological infrastructure—once seen as key to reform 2 – 4 —these efforts have proven inadequate for addressing the deeper systemic issues. Johnson refers to this as the “Era of Entanglement,” a period marked by the intertwined demands of clinical care, regulation, and finance that have produced a fragmented, burdensome system. 5 As with the leap from horse-drawn carriages to motorized vehicles, transformative innovation, rather than iterative, is needed to advance healthcare delivery meaningfully. The National Research Council defines this thinking as radical change, where healthcare exploits “qualitatively new ways of conceptualizing and solving health and healthcare problems and revolutionary ways of addressing these problems.” 4 However, these researchers must understand how healthcare works — they need access to data and observations that spur new hypotheses and ideas. The conundrums they face are the challenges our current environment presents for non-medical faculty to obtain access to patient care settings, due to hospital and privacy policies. Video as a data collection tool has been underutilized in healthcare due to inherent barriers, including confidentiality, privacy, and the technical challenges associated with sharing large file sizes. 6 – 8 Earlier efforts to capture clinical visit knowledge recognized that tacit knowledge, non-verbal interpersonal interactions, and the necessity of understanding ‘embodied cognition’ all emphasize the need to video-capture interactions between physicians and their patients in a natural setting. 6 , 9 – 11 These data also enable the documentation of activities in their entirety and complexity. 12 It builds on the work of others who have used videotapes and eye-tracking methods. 13 – 15 : Video capture alone, however, is an incomplete source of clinic visit details, due to the increasing role of electronic health records, web applications, and other in-room technologies. 16 A systematic review of 175 studies found that only 11% examined the role of computer or electronic medical record use during the encounter. 7 However, there is at least a 35-year corpus of studies of primary care visits that combine qualitative and quantitative research methods. 16 – 25 While these did not result in systematic databases, they often stressed the need for such structures. 26 – 28 The National Academies Press published a major compendium of these efforts. 16 Combining video data with a view of the activities on the computer (EHR logs, clinical notes, and visit-related surveys) as well as room layouts showing the position of the technology with respect to the patient provides a multifaceted view, allowing for qualitative and quantitative analysis and a view supporting a holistic analysis of encounters. This perspective is especially intriguing given current advances in multimodal AI. 29 – 35 There are also concerns that the process of video recording may bias the encounter, thereby lessening the value of these recordings due to the Hawthorne effect. 36 Interestingly, video recordings are shown to be less disruptive to natural participant behavior than human observers present during visits. 6 Another study examining physicians’ consultation behavior found no evidence that awareness of video recording affects objective measures of their performance, supporting the use of videos as valuable tools for teaching and research. 36 To address the challenges of researcher access to clinic encounters and to embrace the need for multimodal data, we created the Observer dataset, a multimodal resource for multidisciplinary research. This paper describes the efforts and observational frameworks, as well as initial studies and observations, enabled by the dataset. Materials and Methods The overarching goal of Observer is to provide a library of clinical exam rooms and interactions among patients, any other caregivers, the primary care provider (PCP), and technology. Our goal is to provide a downloadable dataset for users interested in research directly related to clinical exam rooms. We anticipated that researchers, educators, and policymakers would be the primary audience for this resource. Ultimately, our plan is for Observer to be housed in a repository that adheres to FAIR principles 37 as defined below: Findable by providing metadata that clearly describes the data using a dashboard, being indexed in searchable repositories, and using a digital object identifier to provide a persistent link to its internet location. Accessible by providing the website URL ( https://observer.med.upenn.edu/ ) and data availability statement on a series of public sites, in addition to our lab site. Interoperable by mapping medical data to standard vocabularies using the OMOP table schema for Observer data. Reusable by providing clear usage licenses and documenting information provenance, including any sources for derivative metadata. The remainder of this paper describes the dataset creation process and initial observations. Of note, this paper focuses on our first 100 recorded visits. We continue to record visits, intending to collect 200 visits in adult primary care settings by the end of 2026. Design Considerations We began designing Observer by consulting with two steering groups of adult primary care stakeholders. The first consisted of adult patients and family members. Following a brief presentation by the PI, the group responded to a series of driving questions. They were all very supportive of the project and excited about how it might be used to improve their interactions with physicians and advanced practice nurses. They made the following recommendations for our recruitment and data collection plans: Identify that we are Penn Medicine researchers working with their clinic team (and not, for example, billing or marketing personnel). Compensation for time and effort should be in the form of a $50 physical gift card, rather than one that requires a bank account or an email address. Allow the patient to wear an egocentric (forehead-mounted) camera. Our initial plan was only to ask the PCP to wear this camera, to provide eye tracking as well as a view of the EHR screen. This focus group suggested that observing the patient’s experiences would be invaluable. Strongly consider asking patients and their families for feedback on their visit. Our initial plan was only to ask clinicians for feedback, but patients immediately recognized our plan did not include them, which might cause us to miss their valuable insights. We agreed to incorporate a post-visit patient survey. A second set of focus groups consisted of clinic physicians and nursing staff. After a short presentation by the PI, the group discussion led to the following recommendations. Consider using an after-visit clinician survey instead of requiring clinicians to review their visit recordings and comment on the visit. Clinicians were uncomfortable seeing themselves on the video. To minimize clinic disruption, begin by recording only a few visits for each consented clinician, and then progress to recording multiple visits after staff and consented clinicians are comfortable with the processes. To avoid the risk of new patients losing trust in the clinic team, confirm which patients we would approach until the consented clinicians and clinic managers were comfortable with the study processes. Based on these conversations and a review of similar medicine-related data platforms, 13 – 15 , 38 – 40 we constructed a data plan for Observer that holistically covered the activities of a typical ambulatory care visit. We subsequently obtained IRB approval for visit recording and data collection using the process described below. As noted, this dataset consists of ambulatory care visit videos linked to the EHR user action log, encounter summary, current and past diagnoses, problems, medications, allergies, and all visit-related correspondence from 24 hours before until 48 hours after the recorded visit. Transcripts and final progress notes accompany each of these videos. Our review of existing efforts to create ambulatory visit data sets was disappointing. In particular, none of these platforms supported the retention of multimodal PHI, which requires sophisticated tools for removing identifiers across modalities (facial recognition blurring, voice alteration, and metadata scrubbing) that go far beyond standard text-based de-identification 41 , 42 and extensive curation. Therefore, we realized that to share these data, we would need to create a de-identification pipeline, described below. Data Model To support interoperability with existing EHR-based data and less commonly available video-derived data, we developed a custom data model inspired by OMOP and MIMIC to structure our EHR-derived fields, visit metadata, and survey instruments. Equipment Selection We tested numerous cameras, microphone setups, and screen capture software tools under simulated conditions before selecting our current equipment for collecting encounter information. After tests and discussions with health system security experts, we ruled out using screen-capture software due to the difficulty in aligning those data with room videos, challenges with some software capturing video from more than one application (such as from both the EHR and a web browser) and other equipment challenges, (e.g., bringing in laptops with enterprise licenses for PCPs to use during the visit). After testing various configurations, we selected the following: Room-view camera with a microphone array and a 360-degree room view, connected to a local Zoom session, monitored by a research assistant outside the room. Camera worn on the head at eye level, providing both a microphone and a video from the wearer’s perspective (egocentric view), offered to both the PCP and the patient. For telemedicine encounters, we selected Zoom, which supported secure video downloads, and with which clinical teams and patients were already familiar. To standardize and facilitate remote consent for patients, we initially used an established secure messaging platform. We also learned that linking EHR use to the line-by-line transcripts of PCP-patient interactions is a valuable alignment for end users interested in understanding, for example, what is being discussed while the computer is in use. Previous approaches to video-capturing clinical interactions, such as those by Montague and Asan, 43 relied on multiple fixed cameras positioned at different angles to capture gaze direction. We instead incorporated egocentric head-mounted cameras for both patients and physicians to capture first-person perspectives of what each participant sees during the visit. We also built on the previous work of others in this space, including Senathirajah and colleagues, 13 whose scoping review confirmed that automated detection of eye gaze without eye tracking was feasible and adequate; and Khairat and colleagues, 14 who found that user experience can be measured using surveys for physician satisfaction, system usability, and workload intensity. These latter observations motivated the design of our physician survey. More technical details may be found at https://observer.med.upenn.edu/ . Recruitment and Data Collection Process Our study recruitment focused on patients and PCPs in adult primary care clinics throughout the University of Pennsylvania Health System, an integrated delivery system. We adopted the suggestion of our multistakeholder steering group and required that PCPs be enrolled first, followed by a convenience sample of their patients. Our final recruitment strategy is below and summarized in Supplemental Figure 1. All recruitment materials and any changes made in our protocol are available at https://observer.med.upenn.edu/ . Primary Care Provider Enrollment We presented the project’s goals to primary care providers during their weekly team meetings. After the presentation and question-answering, we provided informed consent forms (ICFs) to each PCP. The ICFs also included options for the degree of de-identification each PCP required for broad consent. Initially, consented PCPs were contacted to establish their preference for days and times best suited for video recording. This process was relaxed once PCPs and we became more comfortable with the study protocol. Patient Enrollment Our study team contacted any patients who were scheduled for visits within consented PCP’s date and time preferences, described the study, and reviewed the consent form using an IRB-approved script. Patients who were willing to participate then received a link to the consent form via email or text message with an option to electronically sign the form or were met by our team to review and sign the ICF before the visit. As offered to providers, patients also had the option to blur their face and/or obscure their voice, with an additional option to blur any other specified body parts. Patients were not recorded if ICFs were not signed before any actual visits. The University of Pennsylvania IRB approved this process. Exam Room Preparation and Visit Data Collection Before visits, a research coordinator set up in-room cameras, then met patients in the room to help them place and activate the egocentric camera. Any additional people accompanying the patient signed an informed consent to be recorded. They had the option of blurring their image and audio. After visits, the research coordinator collected all recording equipment and asked patients to complete the short survey about the visit while still in the room (under 5 minutes on average). Once the surveys were completed, the research coordinator provided patients with gift cards and escorted them out of the exam room. Primary care providers were given an exit survey to complete and return to the research team. They received a gift card after returning the exit survey. EHR Data Extraction After the visit, a research coordinator completed collecting all data for the visit. Recorded videos were copied to a secure cloud platform (Microsoft Azure Databricks) for de-identification and removal of extraneous events, such as turning on the video recorders, which were already captured by the recorders. Survey results were labeled with a visit ID and stored electronically in a secure location. A request was made to our data and analytics core to extract all pertinent data from the EHR and to label these data with the same visit ID. These data were provided to us without PHI and with dates shifted. We updated all data to reflect this new date and ID and stored these data in our same secure location. All other data derived from the visit video were managed by us, with labels provided by our honest broker. All other labels were removed before the data were available for review. Privacy Preservation To preserve privacy, we developed and implemented MedVidDeID, 44 a modular pipeline designed to de-identify clinical audio-video (AV) data in compliance with HIPAA Safe Harbor standards. 45 This system integrates state-of-the-art, open-source tools into a six-stage workflow that treats audio and video modalities independently. First, we extract transcripts using WhisperX and de-identify them with a local instance of PHIlter. Audio was de-identified by redacting timestamped PHI segments and transforming voice characteristics to obscure speaker identity. Video frames are processed using YOLOv11 to detect and blur facial features, utilizing pose estimation and Kalman filtering. The de-identified audio and video are then recombined and subjected to manual quality control to ensure completeness of PHI removal. An analysis of our first 11 hours of clinical encounters demonstrated a 91% video de-identification rate, reducing the need for manual processing time by over 60%. This package enables scalable, secure sharing and analysis of AV clinical data while preserving patient privacy. Results Participant Recruitment We recruited 13 PCPs from 4 clinic settings. We approached 210 patients scheduled to see these PCPs to participate in this study. A total of 129 patients agreed to participate (61%), from which we were able to capture clinic visits for 100 (78%). There were 29 patients who agreed to participate but were not recorded due to illness, rescheduling, or logistical issues. Three patients participated more than once for separate visits. We contacted three patients whose primary language was not English, of whom two participated in the study (data not shown). Figure 1 summarizes enrolled patients self-reported health and educational levels. Download figure Open in new tab Figure 1 Patient characteristics. Data only available for the last 70 visits, due to the addition of these questions on the post-visit surveys. Tables 1 and 2 summarize the results of our first 100 visits. Visit types, as extracted from EHR data, represent the range of visit types for primary care. View this table: View inline View popup Download powerpoint Table 1: Visit characteristics. View this table: View inline View popup Download powerpoint Table 2 Data about primary care provider (PCP) characteristics were obtained via self-reported post-visit surveys Possible answers to the self-reported technical competence question (“How would you rate your comfort level with using technology in healthcare?”) ranged from being a novice (“Initial help”) to being comfortable with technology (“Self-taught”) to being considered an expert (“Teacher”). PCPs were uniformly distributed across years in practice, ranging from 1 to 5 years to more than 20 years in practice. In response to the question, “How would you rate your comfort level with using technology in healthcare?” most providers described themselves as either self-taught or needing instruction (see Table caption for definitions.) Visit length ranged from 5 to 100 minutes. PCPs rated their visit satisfaction as a weighted average of 3.95/5 across all visits, while their patients rated visits as 4.24/5 on average, with occasionally discrepant scores provided by the patient and clinician for the same visit (data not shown, but available on the Observer website). Overall, the visit experience was unaffected by the presence of video recording technology. Of those who perceived some impact, one was positive, where the PCP performing a physical exam reminded a medical student that it was always important to examine the patient without a shirt on. Another PCP reported a negative impact without specifying details. Observer Composition and Structure At the time of this publication, the Observer dataset comprises 110 outpatient primary care visits, each recorded using a standardized protocol. For each visit, Observer includes: Room-view video recordings , captured with a stationary 360-degree camera equipped with a directional microphone array, are positioned to provide a wide-angle perspective of the physical interaction between patient and clinician. Egocentric videos , recorded using head-mounted cameras worn by patients and PCPs. These recordings offer a first-person perspective of visual attention, body language, and interpersonal engagement. They also provide the equivalent of eye-tracking, enabling a view of how computer technology is used during visits. EHR-derived audit logs , including timestamped records of all user interactions within the electronic health record during the visit and the surrounding 72-hour window (24 hours before through 48 hours after the encounter). Structured metadata , covering reason for visit, patient demographics, visit type, clinician characteristics, room configuration, and survey responses. Survey results from patients and clinicians after the visit, capturing perceptions of communication quality, comfort, and the perceived impact of recording technology. Transcripts , automatically generated and manually verified, reflecting verbal exchanges between participants. Clinical notes, including timestamped text entries from providers linked to specific patient visits, capturing detailed observations, diagnoses, and care plans, with metadata on note type and status. Vital signs and measurements, recorded during visits, including quantitative data such as blood pressure, weight, height, pulse, and oxygen saturation, linked to the corresponding patient encounter. To support scalable reuse, we provide standardized file formats (MP4 for video, CSV for logs and metadata, and JSON for annotations) along with accompanying documentation that describes data provenance, schema definitions, and known quality attributes. All data packages are stored in a secure institutional archive, with download access granted through a verification process modeled after the MIMIC framework. Clinic Encounter Visit Details Figure 2 summarizes the visit diagnoses covered in our first 100 recordings. Patients had multiple active diagnoses with a frequency representative of a typical adult primary care setting. Download figure Open in new tab Figure 2 Summary of visit diagnoses. Figure 3 provides a sample of the different room configurations. We observed that the physical configuration of exam rooms and the EHR screens significantly affects the ability of PCPs and patients to see each other face-to-face and to view the EHR. Download figure Open in new tab Figure 3 Sample room configurations of participating clinical sites. Discussion We report our experience in creating the Observer dataset — a first-of-its-kind library of visit recordings, EHR data, and various other forms of accompanying information. Previous efforts, such as the Activity Log Files Aggregation toolkit, while methodologically sound, lacked the infrastructure for long-term data sharing and became inaccessible over time, requiring subsequent studies to collect data independently rather than build upon existing foundations —a limitation that Observer aims to address. We anticipate Observer will enable hypothesis generation and testing, as well as an evolving, valuable dataset for model development and testing. Ethnographic researchers will be able to use Observer to evaluate compliance with healthcare delivery recommendations, observe care provider-patient interaction, and identify opportunities to improve both needed resources and cognitive support for both patients and PCPs. Room configuration varied significantly across different clinics and is likely an essential covariate in studies of communication, as noted by Kumarapel 46 and summarized in a systematic review. 47 For this reason, we added an image of the room configuration and metadata about this configuration to each visit’s data set. These videos provide natural experiments to observe how screen and room arrangements affect the PCPs’ time spent looking at EHR screens versus looking at patients, and how PCPs utilize EHR data in dialogues and explanations. In addition to hypothesis generation and testing, 3 these recordings allow a complete record of consultations to be viewed repeatedly for inter-observer reliability testing and open science. 48 Following discussions with researchers at recent meetings, we believe the multimodal nature of Observer can be beneficial for the classes of problems outlined, as examples in Table 3 (below). View this table: View inline View popup Table 3 Example Use Cases for the Observer Dataset and Their Required Data Modalities. A key contribution of the Observer dataset is the ability to synchronize observational data with process data extracted from EHR logs. These data can serve as powerful new sources for Audit and Feedback (A&F), as they help generate the actionable plans with specific advice for improvement that are known to make feedback most effective. 49 This dataset, along with its ability to generate specific and actionable feedback, helps catalyze multimodal AI applications in various contexts. For example, we are developing a model for identifying agenda topics using de-identified visit transcripts from Observer. 50 Another team is using multimodal data to develop feature-detection models that predict mild cognitive impairment. These data also support model fine-tuning and provide examples for few-shot prompting of large language models. Other contexts will likely involve clinical decision support rules, providing real-time visit information, an almost infinite array of clinical care improvements, physician training, the use and design of EHRs, the role of AI, patient-facing chatbots, and exam room redesign. Limitations and Challenges Although we envision many uses for Observer, we recognize that any data resource of this type suffers from limitations that are either measurable or implied. For example, we have observed no systematic skew in recruiting physicians or patients; however, physician liability concerns could introduce a selection bias in their participation, resulting in a biased sample of patients invited to participate. These potential biases necessitate vigilant monitoring and the implementation of measures to ensure the collection of representative samples. The ethical implications of video recording and the possible risks to participant confidentiality and privacy must be meticulously addressed to maintain the integrity of the research and the trust of the participants. Balancing the benefits of rich, detailed data collection with the ethical considerations and participants’ comfort is crucial in leveraging representative video recordings for research purposes. 51 , 52 The request for video recordings may have raised concerns among potential participants. Although we did not collect data from non-respondents, some individuals may have been reluctant to participate because of the possibility that a) multiple individuals may view the video recordings over time, b) unauthorized individuals may access video data due to improper storage, and c) an individual’s identity could be more easily discerned from a video recording than from written data. Although we emphasized privacy protections and included clear language about participants’ right to withdraw at any time, some patients still expressed concerns that influenced their willingness to participate. We provided modest gift cards as a token of appreciation, and we allowed patients to select the level of privacy preservation they required to participate, as part of the informed consent process. These strategies seemed to support participation and may be helpful for others collecting encounter video data. Conclusion Observer builds on previous work to create a data resource containing a completely de-identified view of ambulatory care visits—with multiple camera views, a visit transcript, an EHR audit log covering from 24 hours before the visit until 48 hours after the visit, and all pertinent EHR data used or accessible during the visit. Creating the Observer dataset required attention to a holistic view of a clinical encounter, as well as ensuring patient and PCP privacy using state-of-the-art techniques. Next steps will focus on developing the dataset into a repository available for general use. Data Availability All data produced are available online at https://observer.med.upenn.edu/ https://observer.med.upenn.edu/ https://github.com/kbjohnson-penn/observer_frontend Data Availability The Observer dataset is available at https://observer.med.upenn.edu/ . Requests for access to its data may be made by following the instructions on that website. Code Availability All code related to this project is available on GitHub, at https://github.com/kbjohnson-penn/observer_frontend Author Contributions All authors contributed to the conception and design of this project. Mr. Mopidevi provided the technical work leading to this manuscript and contributed to the writing of the Methods section. Mr. Alasaly and Dr. Johnson analyzed the data reported in this manuscript. Drs. Koppel, Jang, and Eaton contributed to the interpretation of the data reported in this manuscript. All authors approve the submitted version of this manuscript. Competing Interests None Grant Support This work was supported by the National Library of Medicine and the NIH Office of the Director under project number 5DP1LM014558-03 (Former Number: 1DP1OD035237-01) for the project entitled “Helping Doctors Doctor: Using AI to Automate Documentation and ‘De-Autonomate’ Health Care.” Acknowledgements We appreciate the thoughtful review of this manuscript by Dr. S. Trent Rosenbloom and Dr. John Holmes, as well as the proofreading of this manuscript by Lauren Malloy. Footnotes rkoppel{at}sas.upenn.edu Based on informative feedback, this manuscript has been revised for clarity. The main focus is on the creation of the Observer dataset, our unique multi-modal library of video-recorded clinic visits linked with all related metadata. We have strengthened the background to demonstrate the research gap our dataset fill. We now acknowledge an extensive 35-year corpus of qualitative and quantitative studies in primary care, emphasizing that while these studies stressed the need for systematic data structures, they did not result in shareable databases. We also included data from a recent systematic review showing a lack of prior work that integrates video observation with analysis of EHR use, as well as the technical difficulties in sharing such large, multimodal data. Funding details have been added at the end of the manuscript. References 1. ↵ Blumenthal D , Shah , A , Gunja , MZ , Williams , II RD . Mirror, Mirror 2024: A Portrait of the Failing U.S. Health System . Accessed April 29, 2025. https://www.commonwealthfund.org/publications/fund-reports/2024/sep/mirror-mirror-2024 2. ↵ Stead WW . Rethinking electronic health records to better achieve quality and safety goals . Annu Rev Med . 2007 ; 58 : 35 – 47 . doi: 10.1146/annurev.med.58.061705.144942 OpenUrl CrossRef PubMed Web of Science 3. ↵ Johnson KB , Stead WW . Making Electronic Health Records Both SAFER and SMARTER . JAMA . 2022 ; 328 ( 6 ): 523 – 524 . doi: 10.1001/jama.2022.12243 OpenUrl CrossRef PubMed 4. ↵ National Research Council (US) Committee on Engaging the Computer Science Research Community in Health Care Informatics . Computational Technology for Effective Health Care: Immediate Steps and Strategic Directions . ( Stead WW , Lin HS , eds.). National Academies Press ( US ); 2009 . Accessed June 13, 2022. http://www.ncbi.nlm.nih.gov/books/NBK20640/ 5. ↵ Johnson KB , Neuss MJ , Detmer DE . Electronic health records and clinician burnout: A story of three eras . J Am Med Inform Assoc . 2021 ; 28 ( 5 ): 967 – 973 . doi: 10.1093/jamia/ocaa274 OpenUrl CrossRef PubMed 6. ↵ Asan O , Montague E . Using video-based observation research methods in primary carehealth encounters to evaluate complex interactions . Journal of Innovation in Health Informatics . 2014 ; 21 ( 4 ): 161 – 170 . doi: 10.14236/jhi.v21i4.72 OpenUrl CrossRef PubMed 7. ↵ Golembiewski EH , Espinoza Suarez NR , Maraboto Escarria AP , et al. Video-based observation research: A systematic review of studies in outpatient health care settings . Patient Educ Couns . 2023 ; 106 : 42 – 67 . doi: 10.1016/j.pec.2022.09.017 OpenUrl CrossRef PubMed 8. ↵ Gilmore RO , Adolph KE , Millman DS , Gordon A . Transforming Education Research Through Open Video Data Sharing . Adv Eng Educ . 2016 ; 5 ( 2 ): http://advances.asee.org/wp-content/uploads/vol05/issue02/Papers/AEE-18-Gilmore.pdf . 9. ↵ Chan S . Using videos and multimodal discourse analysis to study how students learn a trade . International Journal of Training Research . 2013 ; 11 ( 1 ): 69 – 78 . doi: 10.5172/ijtr.2013.11.1.69 OpenUrl CrossRef 10. Coleman T . Using video-recorded consultations for research in primary care: advantages and limitations . Fam Pract . 2000 ; 17 ( 5 ): 422 – 427 . doi: 10.1093/fampra/17.5.422 OpenUrl CrossRef PubMed Web of Science 11. ↵ Nassauer A , Legewie NM . Video Data Analysis: A Methodological Frame for a Novel Research Trend . Sociological Methods & Research . 2021 ; 50 ( 1 ): 135 – 174 . doi: 10.1177/0049124118769093 OpenUrl CrossRef 12. ↵ Schaeffer JH . Videotape: New Techniques of Observation and Analysis in Anthropology . In: Hockings P , ed. Principles of Visual Anthropology . DE GRUYTER ; 1995 : 255 – 284 . doi: 10.1515/9783110290691.255 OpenUrl CrossRef 13. ↵ Senathirajah Y , Borycki EM , Kushniruk A , Cato K , Wang J . Use of Eye-Tracking in Studies of EHR Usability - The Current State: A Scoping Review . Stud Health Technol Inform . 2019 ; 264 : 1976 – 1977 . doi: 10.3233/SHTI190742 OpenUrl CrossRef PubMed 14. ↵ Khairat S , Coleman C , Ottmar P , Bice T , Carson SS . Evaluation of Physicians’ Electronic Health Records Experience Using Actual and Perceived Measures . Perspect Health Inf Manag . 2022 ; 19 ( 1 ): 1k . OpenUrl 15. ↵ Khairat S , Coleman C , Ottmar P , Jayachander DI , Bice T , Carson SS . Association of Electronic Health Record Use With Physician Fatigue and Efficiency . JAMA Netw Open . 2020 ; 3 ( 6 ): e207385 . doi: 10.1001/jamanetworkopen.2020.7385 OpenUrl CrossRef 16. ↵ Committee on Implementing High-Quality Primary Care , Board on Health Care Services , Health and Medicine Division , National Academies of Sciences , Engineering, and Medicine . Implementing High-Quality Primary Care: Rebuilding the Foundation of Health Care . ( McCauley L , Phillips RL , Meisnere M , Robinson SK , eds.). National Academies Press ; 2021 : 25983 . doi: 10.17226/25983 OpenUrl CrossRef 17. Crabtree BF , Miller WL , eds. Doing Qualitative Research . Second edition . SAGE ; 2012 . 18. Westfall JM , Mold J , Fagnan L . Practice-based research--“Blue Highways” on the NIH roadmap . JAMA . 2007 ; 297 ( 4 ): 403 – 406 . doi: 10.1001/jama.297.4.403 OpenUrl CrossRef PubMed Web of Science 19. Westfall JM , Fagnan LJ , Handley M , et al. Practice-based research is community engagement . J Am Board Fam Med . 2009 ; 22 ( 4 ): 423 – 427 . doi: 10.3122/jabfm.2009.04.090105 OpenUrl FREE Full Text 20. Westfall JM , Zittleman L , Staton EW , et al. Card studies for observational research in practice . Ann Fam Med . 2011 ; 9 ( 1 ): 63 – 68 . doi: 10.1370/afm.1199 OpenUrl Abstract / FREE Full Text 21. Westfall JM , Roper R , Gaglioti A , Nease DE . Practice-Based Research Networks: Strategic Opportunities to Advance Implementation Research for Health Equity . Ethn Dis . 2019 ; 29 ( Suppl 1 ): 113 – 118 . doi: 10.18865/ed.29.S1.113 OpenUrl CrossRef PubMed 22. Braithwaite J . Growing Inequality: Bridging Complex Systems, Population Health and Health Disparities . International Journal of Epidemiology . 2018 ; 47 ( 1 ): 351 – 353 . doi: 10.1093/ije/dyy001 OpenUrl CrossRef 23. Stange KC , Miller WL , Crabtree BF , O’Connor PJ , Zyzanski SJ . Multimethod research: approaches for integrating qualitative and quantitative methods . J Gen Intern Med . 1994 ; 9 ( 5 ): 278 – 282 . doi: 10.1007/BF02599656 OpenUrl CrossRef PubMed Web of Science 24. Stange KC , Miller WL , McWhinney I . Developing the knowledge base of family practice . Fam Med . 2001 ; 33 ( 4 ): 286 – 297 . OpenUrl PubMed Web of Science 25. ↵ Stange KC , Zyzanski SJ . Integrating qualitative and quantitative research methods . Fam Med . 1989 ; 21 ( 6 ): 448 – 451 . OpenUrl PubMed 26. ↵ Aungst H , Baker M , Bouyer C , et al. Identifying Personal Strengths to Help Patients Manage Chronic Illness . Patient-Centered Outcomes Research Institute (PCORI) ; 2019 . Accessed September 22, 2025. http://www.ncbi.nlm.nih.gov/books/NBK595698/ 27. Macaulay AC , Commanda LE , Freeman WL , et al. Participatory research maximises community and lay involvement . BMJ . 1999 ; 319 ( 7212 ): 774 – 778 . doi: 10.1136/bmj.319.7212.774 OpenUrl FREE Full Text 28. ↵ Homa L , Rose J , Hovmand PS , et al. A participatory model of the paradox of primary care . Ann Fam Med . 2015 ; 13 ( 5 ): 456 – 465 . doi: 10.1370/afm.1841 OpenUrl Abstract / FREE Full Text 29. ↵ Bokade R , Navato A , Ouyang R , et al. A cross-disciplinary comparison of multimodal data fusion approaches and applications: Accelerating learning through trans-disciplinary information sharing . Expert Systems with Applications . 2021 ; 165 : 113885 . doi: 10.1016/j.eswa.2020.113885 OpenUrl CrossRef 30. Bundy H , Gerhart J , Baek S , et al. Can the Administrative Loads of Physicians be Alleviated by AI-Facilitated Clinical Documentation? J Gen Intern Med . Published online June 27, 2024 . doi: 10.1007/s11606-024-08870-z OpenUrl CrossRef PubMed 31. Reed M , Rampono B , Turner W , et al. A multicentre validation study of a smartphone application to screen hand arthritis . BMC Musculoskelet Disord . 2022 ; 23 ( 1 ): 433 . doi: 10.1186/s12891-022-05376-9 OpenUrl CrossRef PubMed 32. Silverman D. Interpreting Qualitative Data . Seventh . Sage Publishing ; 2024 . 33. Classen DC , Longhurst C , Thomas EJ . Bending the patient safety curve: how much can AI help? NPJ Digit Med . 2023 ; 6 ( 1 ): 2 . doi: 10.1038/s41746-022-00731-5 OpenUrl CrossRef PubMed 34. Acosta JN , Falcone GJ , Rajpurkar P , Topol EJ . Multimodal biomedical AI . Nat Med . 2022 ; 28 ( 9 ): 1773 – 1784 . doi: 10.1038/s41591-022-01981-2 OpenUrl CrossRef PubMed 35. ↵ Xue C , Kowshik SS , Lteif D , et al. AI-based differential diagnosis of dementia etiologies on multimodal data . Nat Med . 2024 ; 30 ( 10 ): 2977 – 2989 . doi: 10.1038/s41591-024-03118-z OpenUrl CrossRef 36. ↵ Pringle M , Stewart-Evans C . Does awareness of being video recorded affect doctors’ consultation behaviour? Br J Gen Pract . 1990 ; 40 ( 340 ): 455 – 458 . OpenUrl Abstract / FREE Full Text 37. ↵ Wilkinson MD , Dumontier M , Aalbersberg IjJ , et al. The FAIR Guiding Principles for scientific data management and stewardship . Sci Data . 2016 ; 3 ( 1 ): 160018 . doi: 10.1038/sdata.2016.18 OpenUrl CrossRef PubMed 38. ↵ Harris PA , Taylor R , Thielke R , Payne J , Gonzalez N , Conde JG . Research electronic data capture (REDCap)--a metadata-driven methodology and workflow process for providing translational research informatics support . J Biomed Inform . 2009 ; 42 ( 2 ): 377 – 381 . doi: 10.1016/j.jbi.2008.08.010 OpenUrl CrossRef PubMed Web of Science 39. Gilmore RO , Adolph KE , Millman DS . Curating identifiable data for sharing: The databrary project . In: 2016 New York Scientific Data Summit (NYSDS) . IEEE ; 2016 : 1 – 6 . doi: 10.1109/NYSDS.2016.7747817 OpenUrl CrossRef 40. ↵ Liberman , M , Cieri C. The Evolution of the Linguistic Data Consortium . Accessed January 22, 2024 . https://scholar.google.com/scholar_url?url= https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/lrec1998-creation-distribution-use.pdf&hl=en&sa=X&ei=8ISuZf2RKcrJy9YPkL-OiAY&scisig=AFWwaeYm4hUhOlQ1bH2e8Q1mjVoG&oi=scholarr 41. ↵ Talby D. Delivering Regulatory-Grade, Automated, Multimodal Medical Data De-Identification . September 7, 2025 . https://www.johnsnowlabs.com/delivering-regulatory-grade-automated-multimodal-medical-data-de-identification/ 42. ↵ Anibal J , Huth H , Li M , et al. Voice EHR: introducing multimodal audio data for health . Front Digit Health . 2025 ; 6 . doi: 10.3389/fdgth.2024.1448351 OpenUrl CrossRef 43. ↵ Montague E , Asan O . Dynamic modeling of patient and physician eye gaze to understand the effects of electronic health records on doctor-patient communication and attention . Int J Med Inform . 2014 ; 83 ( 3 ): 225 – 234 . doi: 10.1016/j.ijmedinf.2013.11.003 OpenUrl CrossRef PubMed 44. ↵ Mopidevi VSSH , Jang KJ , Alasaly B , et al. MedVidDeID: Protecting Privacy in Clinical Encounter Video Recordings . SSRN. Preprint posted online 2025 . doi: 10.2139/ssrn.5194782 OpenUrl CrossRef 45. ↵ Guidance Regarding Methods for De-Identification of Protected Health Information in Accordance with the Health Insurance Portability and Accountability Act (HIPAA) Privacy Rule . U.S. Department of Health and Human Services Accessed May 12, 2025. https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html 46. ↵ Kumarapeli P , de Lusignan S . Using the computer in the clinical consultation; setting the stage, reviewing, recording, and taking actions: multi-channel video study . J Am Med Inform Assoc . 2013 ; 20 ( e1 ): e67 – 75 . doi: 10.1136/amiajnl-2012-001081 OpenUrl CrossRef PubMed 47. ↵ Patel MR , Vichich J , Lang I , Lin J , Zheng K . Developing an evidence base of best practices for integrating computerized systems into the exam room: a systematic review . J Am Med Inform Assoc . 2017 ; 24 ( e1 ): e207 – e215 . doi: 10.1093/jamia/ocw121 OpenUrl CrossRef PubMed 48. ↵ Adolph KE , Gilmore RO , Freeman C , Sanderson P , Millman D . Toward Open Behavioral Science . Psychological Inquiry . 2012 ; 23 ( 3 ): 244 – 247 . doi: 10.1080/1047840X.2012.705133 OpenUrl CrossRef PubMed 49. ↵ Ivers N , Yogasingam S , Lacroix M , et al. Audit and feedback: effects on professional practice. Cochrane Central Editorial Service, ed . Cochrane Database of Systematic Reviews . 2025 ; 2025 ( 3 ). doi: 10.1002/14651858.cd000259.pub4 OpenUrl CrossRef 50. ↵ Jang KJ , Bhatti S , Pugh S , et al. Towards a Real-time Clinical Agenda Setting System for Enhancing Clinical Interactions in Primary Care Visits . AAAI GenAI4Health Workshop . Published online 2025. https://openreview.net/forum?id=Lu1dxr1bWV 51. ↵ Maldonado-Castellanos I , Barrios LM . Ethical Issues when Using Digital Platforms to Perform Interviews in Qualitative Health Research . International Journal of Qualitative Methods . 2023 ; 22 : 16094069231165949 . doi: 10.1177/16094069231165949 OpenUrl CrossRef 52. ↵ Sartori L , Theodorou A . A sociotechnical perspective for the future of AI: narratives, inequalities, and human control . Ethics Inf Technol . 2022 ; 24 ( 1 ): 4 . doi: 10.1007/s10676-022-09624-3 OpenUrl CrossRef View the discussion thread. Back to top Previous Next Posted October 03, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research Kevin B. Johnson , Basam Alasaly , Kuk Jin Jang , Eric Eaton , Sriharsha Mopidevi , Ross Koppel medRxiv 2025.05.18.25327837; doi: https://doi.org/10.1101/2025.05.18.25327837 Share This Article: Copy Citation Tools Observer: Creation of a Novel Multimodal Dataset for Outpatient Care Research Kevin B. Johnson , Basam Alasaly , Kuk Jin Jang , Eric Eaton , Sriharsha Mopidevi , Ross Koppel medRxiv 2025.05.18.25327837; doi: https://doi.org/10.1101/2025.05.18.25327837 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Informatics Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a003ee29fb3baa64',t:'MTc3OTUzNzk0Mg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();