The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries

The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries View ORCID Profile Thomas C. Draper , Jason Leake , Kathryn Lamb-Riddell , Timothy Cox , John McCormick , Stephen Trowell , Janice Kiely , Richard Luxton doi: https://doi.org/10.1101/2025.03.24.25324398 Thomas C. Draper a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Research Fellow Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Thomas C. Draper For correspondence: tom.draper{at}uwe.ac.uk Jason Leake a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Researcher Find this author on Google Scholar Find this author on PubMed Search for this author on this site Kathryn Lamb-Riddell a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Senior Research Technician Find this author on Google Scholar Find this author on PubMed Search for this author on this site Timothy Cox a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Research Fellow Find this author on Google Scholar Find this author on PubMed Search for this author on this site John McCormick b NHS England South West, South West House, Blackbrook Ave , Taunton, TA1 2PX, UK Roles: Clinical Lead for Innovation Find this author on Google Scholar Find this author on PubMed Search for this author on this site Stephen Trowell b NHS England South West, South West House, Blackbrook Ave , Taunton, TA1 2PX, UK Roles: Director of Digital Transformation Find this author on Google Scholar Find this author on PubMed Search for this author on this site Janice Kiely a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Professor Find this author on Google Scholar Find this author on PubMed Search for this author on this site Richard Luxton a Centre of Digital Excellence, University of West of England , Bristol, BS16 1QY, UK Roles: Professor Find this author on Google Scholar Find this author on PubMed Search for this author on this site Abstract Full Text Info/History Metrics Supplementary material Data/Code Preview PDF Abstract Objectives To investigate the effect of the introduction of environmental noise, microphone identity & position, and informational noise on the accuracy of a commercial Clinical AI Scribe (CAIS). Design Consultations on five medical conditions (memory loss, diarrhoea, headaches, skin rash, or prostate symptoms) were recorded. A ground-truth audio file was produced for a noise free consultation. This in turn produced a ground-truth clinical summary via the AI scribe, typically containing 30 facts. Different types of ‘noise’ at different levels were then investigated by (i) increasing the distance of the consultation from the microphone, and changing microphone type (ii) introducing background noise: heavy rain, baby crying, construction noise and a toddler at a range of levels between -10 to +5dB relative to the consultation audio and (iii) introducing informational noise: the medical consultations were modified to include additional conversations between the clinician and the patient to introduce irrelevant discussions (medical and non-medical, shorter and longer). The resulting 160 summaries were then compared with the corresponding ground-truth summaries and the errors introduced were classified and enumerated. Setting Simulated primary care setting. Participants The roles of the doctor and patient were played by actors, rotating between roles. Main outcome measures Outcomes measured were the quantification of the degradation of accuracy of the clinical summary, i.e. omissions, hallucinations and inclusions, relative to the ground-truth summary, following the introduction of noise of all types at different levels. Results Error rates and absolute error counts increased for all noise levels and noise types; the principal error being omissions. At 4.5 m from the consultation, for all microphones tested, all facts were omitted. At up to 2 m all microphones, except the laptop, only introduced a small number (<5) of omissions. For background noise, the type of noise was found to correlate with omissions – toddler and heavy rain were particularly deleterious, 7 and 15 omissions at 0 and +5dB respectively, i.e. when the noise loudness was comparable to or greater than that of the consultation. The CAIS was remarkably apt at rejecting additional informational noise. This was true for the addition of either short or long, medical or non-medical, information. Conclusions The CAIS was very good at capturing a doctor-patient consultation and producing accurate clinical summaries, however error rate (especially omissions) increased notably when acoustic noise was introduced. The system was better able to handle informational noise. What is already known on this topic The use of AI is rapidly increasing, with many commercial products available, both specific to clinical use and not. Commercial Clinical AI Scribes (generating clinical summaries from consultation audio) are being used in healthcare settings, sometimes with little guidance or knowledge of employers. These tools can operate well in perfect conditions, but there is limited data on nosier environments. What this study adds The commercial Clinical AI Scribe only introduced a small number of errors when introduced to informational noise, however large numbers of errors (especially omissions) were caused by acoustic noise (dependant on the particular sound type and volume), especially with low-quality poorly-located microphones. Users of Clinical AI Scribes must be particularly vigilant in checking the summary when the environment contains certain deleterious background noises, even at lower volumes, to ensure errors are not introduced to the patient records. Introduction Recent studies have highlighted the transformative potential of AI in healthcare, e.g. see the reviews by Chustecki [ 1 ] and Al Kuwaiti [ 2 ]. The reviews highlight potential advantages of the technology together with risks and challenges, e.g. data security, regulatory issues and acceptance by patients and clinicians [ 3 – 5 ]. The most mature application is probably in the interpretation of diagnostic images, e.g. as produced by MRI, X-ray and CT scans [ 6 , 7 ]. The use of AI as a clinician decision support tool in general practice is also in development [ 8 ], as is the generation of discharge summaries and instructions for patients [ 9 , 10 ]. Recently, there has been a surge in the development of AI powered scribe technologies which transcribe clinician patient consultations into structured medical summaries [ 11 ]. There are now a significant number of such software packages available commercially. Reviews of some of these products are available via the KLAS website ( https://engage.klasresearch.com/ ). Some of these products, together with databases of doctor patient encounters generated by these products, are also described in the bibliography of [ 11 ]. The AI medical scribe [ 12 ] adopts a two-step process (i) automatic speech recognition (ASR) produces a text transcript from the digital audio input which in turn is (ii) processed by a large language model (LLM) to produce a clinical summary. The summary is then input to the patient’s electronic health record (EHR). As there can be errors present in these summaries, it is currently mandated by commercial suppliers that the clinician should check the summary and correct for any errors prior to input to the patient’s EHR. It has been proposed that the quality of the consultation is improved as the clinician does not need to make notes during the consultation and so is able to maintain improved eye contact with the patient [ 13 ]. The automated approach also has the potential to reduce the administrative burden on clinicians [ 14 ], thereby addressing the issue of burnout among clinicians [ 15 ]. It has also been reported [ 16 ] that the AI scribe can reduce the risk of errors which might be introduced by the clinician using manual input and increase the efficacy of EHR management. Significant errors may be introduced by AI scribes; see for example Kernberg et al [ 17 ] who report significant errors for a ChatGPT-4 based scribe. Methods to determine and classify errors have recently been reviewed by Gebauer [ 11 ]. These include automated approaches, e.g. ROUGE and BERT [ 18 – 21 ] and also manual approaches [ 17 ] in which the clinical notes are compared with the original audio file, often by clinically trained personnel. A number of other metrics are also described, e.g. a modified Physician Documentation Quality Instrument-9 (PDQI-9) [ 22 ] and the Sheffield Assessment Instrument for Letters (SAIL) [ 23 , 24 ] approaches. Despite the potential advantages, significant additional challenges exist in a real world clinical scenario in obtaining an accurate clinical summary of the consultation. The investigation of these challenges form the focus of the current paper: (i) firstly, the placement (and quality) of the microphone is critical to obtain an optimal audio input to the ASR; (ii) secondly, the clinical environment can be noisy with background noise present, e.g. from building work outside or the presence of a crying baby; (iii) and thirdly, informational noise may also be present during the consultation, e.g. a long and irrelevant diatribe about a hobby, or medical discussion which is not relevant to the consultation. These and other challenges have been described by Quiroz et al [ 25 ]. The effect of these additional challenges is investigated here experimentally. The approach that we have adopted is firstly to establish a ground-truth clinical summary with a good quality, noise-free audio file recording of a consultation, which is input to the Clinical AI Scribe. Noise, either acoustic (microphone or background) or informational (medical or non-medical), is then introduced at different levels to the audio file for the three challenges outlined above. The altered audio file is then presented to the Clinical AI Scribe to produce a modified clinical summary, which is then compared to the ground-truth summary to establish the additional errors introduced by the presence of the introduced noise. Methods Study Design All medical consultations used, recorded, or referred to in this study were performed using actors and simulated medical conditions. For full protocol details, see the Protocol file in the supplementary information. This research was conducted in a simulated primary care setting at the NHS South West Centre of Digital Excellence (CoDE). This provided a simulated GP surgery, allowing for increased realism in the positioning of persons and equipment, background noise, and acoustic profile. A controlled experimental framework was implemented to assess the impact of microphone cost (RRP £25–£120) and placement, background noise, and varying informational noise (both medical and non-medical) on the accuracy of a widely-used commercial Clinical AI Scribe. A repeated-measures approach was adopted, enabling systematic variation of these factors while maintaining as consistent an input to the AI as possible. Specifically, simulated consultations (typical 10 min in length) were recorded and presented to the clinical AI scribe, which generated a clinical summary (containing typically 25–30 points of information). The original audio files were then modified (for example, by including construction noise or a prolonged conversation about tennis) and re-presented to the same Clinical AI Scribe, which generated a new clinical summary, which could be compared to the original summary. These processes are displayed in figure 1 . Download figure Open in new tab Figure 1: A graphical representation of the study design. (A) The general process for using a clinical AI scribe, from the audio being detected by the microphone through the CAIS, resulting in a clinical summary. (B) The process used to investigate microphones and their positioning. Different microphones, at different positions, were used to record a consultation, then all of the resulting summaries were compared to the original ‘clean’ summary. (C) The process used to investigate different background noises. To the original audio was added various noises at various volumes, which were then presented to the CAIS, the resulting summaries from which were then compared to the original ‘clean’ summary. (D) The process used to investigate varying informational noise. Medical or non-medical monologues or conversations were inserted into the original audio file, which are then presented to the CAIS, the resulting summaries from which were then compared to the original ‘clean’ summary. When comparing clinical summaries, three measures are used: Omissions: These are points of information which are present in the original clinical summary, but not present in the compared summary. Hallucinations: These are points of false information which are not present in the original clinical summary, but are present in the compared summary. Inclusions: These are points of truthful information which are not present in the original clinical summary, but are present in the compared summary. This is similar to the approach of Kernberg et al. [ 17 ], but with the addition of inclusions due to the inter-summary nature of our comparison. Patient and Public Involvement Statement A Public/Patient Group (Citizens Reference Group) has been set up for the NHS Healthcare Centre of Digital Excellence. They were consulted in the design phase of the study and were provided with early-stage results to which they gave feedback to further inform the study. Results Impact of Microphone Placement (Acoustic Noise) The distance between the participants of the consultation and the recording microphone is a key factor in ensuring high quality output from the clinical AI. Four microphone positions were looked at: control, 0.5 m, 2.0 m, and 4.5 m, chosen to simulate a seated conversation, small movement (e.g. to do test), and larger movement (e.g. moving to couch or behind examination screen). As can be seen in figure 2 , the total points of information reduces as the distance is increased. This is expected because as the distance is increased the intensity of the sound signal at the microphone decreases. This might be expected to follow an inverse square law, I ∝ d −2 where I is the audio intensity and d is the distance. The decrease in the intensity results in a reduction in the bit depth of the recorded speech, whilst the bit depth of the background noise remains constant, ultimately resulting in a sharp decrease in the signal-to-noise ratio, and so the accuracy of the text produced by the ASR. Download figure Open in new tab Figure 2: The median points of information, as a percentage, included in a clinical summary produced by an AI with different microphones (having different costs, see Protocol supplementary information), as a function of distance and relative to the original summary, for all five medical conditions combined. The blue dashed lines are to guide the eye. This decrease in the points of information as the distance increases correlates with the increase in the numbers of errors, as shown in figure 3 . The number of hallucinations and inclusions remains relatively constant across the distances, though the number of omissions rapidly increases, and in many cases the number of omissions equates to the original number of points of information (thus the complete loss of information at 4.5 m, shown in figure 2 ). These omissions occur because of a failure of the speech-to-text (ASR) process, which is unable to accurately recognise the English speech. A lack of inclusions is anticipated, as the introduction of additional truthful information is unlikely when the original information has already been compromised. Of note however, is the small but visible peak in hallucinations at the 2m position. This is primarily caused by the cheapest of the tested microphones, which had a surprising tendency to cause hallucinations at this distance. At 0.5 m, the signal is sufficiently robust to permit operation with minimal errors, whereas at 4.5 m the signal is so weak that the AI is unable to continue processing; however, at 2 m, an “anti-Goldilocks zone” is observed. In this region the LLM assumes a sufficient quantity and quality of information to proceed, but is overconfident. Download figure Open in new tab Figure 3. The number and types of errors at different microphone positions, d: 0.5 m, 2.0 m, and 4.5 m from the midpoint of the source, for all microphones and medical conditions combined. It is interesting to note that, in most cases, the cost/brand of the microphone has little impact on the performance of the clinical AI, with all but one sharply dropping off after the distance is increased beyond 2 m. The sole exception to this is the laptop integrated microphone, which deteriorated at a much greater rate. This can also be seen in figure S1 in the supplementary information. Impact of Background (Acoustic) Noise In a real-world primary care setting, there could be significant confusing sounds present, e.g. generated by those present in the consultation and also outside the consulting room. Four different typical background noises were investigated at varying volumes: baby crying, construction noise, heavy rain, and toddler chatter (for source details, see supplementary information). All four resulted in the introduction of errors, with the number of errors typically increasing as the volume of the background noise, relative to the consultation sound level, was increased. This increase can be seen in figures 4 and S2 (supplementary information). The volumes used were 0 dB (approximately the same volume as the consultation), -10 dB (about half the perceived loudness, or ∼10% of the sound pressure intensity), -5 dB (∼30% of the sound pressure level, i.e. noticeably quieter), and +5 dB (∼3x sound pressure level, i.e. noticeably louder). For reference, the background noise could be clearly discerned at -10 dB, whilst at +5 dB the speech was still comprehensible, though the consultation would be unlikely to proceed until the noise had abated. Moderate rainfall producing noise levels around -5 dB in an office is within the realm of possibility; however, rainfall sufficient to reach +5 dB is less likely. In contrast, internal noise contributions (e.g. from a toddler or baby) have a higher probability of reaching the +5 dB threshold. Download figure Open in new tab Figure 4: The number (and type) of errors introduced when different volumes of the four background noises are introduced. Four different sources of background noise are presented, with results for all medical scenarios combined. Hal: Hallucination, Inc: Inclusions, Omi: Omission. Values shown are the mean, and errors bars are standard error. When comparing directly between volumes, across all background noise sources, there was no statistically significant increase in error rate between the -10 dB and -5 dB samples (p=0.1175), nor between the -10 dB and 0 dB samples (p=0.0782). However, the difference between -10 dB and +5 dB was significant (p=0.0055). At this highest level of noise, the AI system struggled to isolate the clinician’s and patient’s speech, leading to degraded accuracy and incomplete clinical summaries. It is interesting to note that there is a strong correlation between the type of background noise and the number of errors introduced. The introduction of the toddler’s continuous, though unstructured chattering, caused a great number of errors. At the other end of the spectrum, both the baby and construction noise caused far fewer errors. As there was no recognisable speech in these noises, the only mechanism for disruption was by literally drowning out the speech. However, the introduction of heavy rain (pink noise) at high volumes caused the greatest number of errors, suggesting that the frequency profile of certain background noises affects AI recognition differently. It also appears that intermittent background sounds, such as loud crashes, were better tolerated by the AI system compared to continuous noises. Impact of Informational Noise The AI generated summary should capture only the important and relevant information. Examples of potentially confounding information are: (i) irrelevant medical-themed monologues by the patient, which should note dilute the clinician’s professional view, and not appear in the clinical summary. (ii) Casual conversations (for example, about the weather) should also be omitted from the clinical summary. Medical and non-medical informational noise was added as either as a short snippet, or a longer rigmarole. Full details are in the supplementary information, but examples include: Patient very worried about unlikely cancer Patient very worried about unlikely chicken pox/shingles A protracted two-way discussion about tennis Patient complaining about the condition of the floor at their badminton club When superfluous information was introduced into the audio file, either as a patient-derived-medical or conversational non-medical form, the clinical AI scribe was remarkably apt at reporting only the relevant details. Conversational pieces (for example, about a tennis club) were entirely left out, whilst irrelevant medical details (or speculative self-diagnoses) were either omitted or in some way tagged as being a patient opinion or concern. Nevertheless, whilst this informational noise was usually not included (or was at least appropriately labelled), some errors still occurred. Figure 5 shows the absolute number of errors introduced into the summaries, whilst the relative error rate can be seen in figure S3 (supplementary information). The presence of a small number of inclusions is expected, as there may be new information to report (e.g. ‘patient worried about brain cancer’). However, the presence of omissions indicates that the proportion of medically relevant information captured has reduced. Throughout all experiments in this section, only a single hallucination was observed (see figure S3), which was merely a note to follow-up on a blood pressure check (the blood pressure check had been discussed, but the specific follow-up had not). Download figure Open in new tab Figure 5: The number of errors (inclusions or omissions) introduced from the introduction of informational noise, from either non-medical conversations (bottom two plots) or patient-derived medical monologues (top two plots), for all medical scenarios combined. For each of these situations the quantity of informational noise is characterised as either short (left two plots) or long (right two plots). Transcripts of the inserted audio can be seen in the supplementary information. Interestingly, there was no significant difference between the number of errors produced from irrelevant medical vs irrelevant non-medical information (p=0.6806, W=477 – Wilcoxon rank sum test with continuity correction). This suggests that the AI scribe is equally capable of rejecting both types of informational noise. Likewise, the difference between shorter and longer segments of informational noise was not significant when looking at omissions (p=0.846, W=463 – Wilcoxon rank sum test with continuity correction). Discussion Interpretation of Findings The findings of this study show that this commercial Clinical AI Scribe can be highly sensitive to environmental conditions, particularly acoustic noise such as caused by microphone positioning or certain background noises. It should be noted however, that the errors produced add onto what is already a (typically very good, though) imperfect summary. We have observed that the commercial Clinical AI Scribe produces on average around 0.5 hallucinations and 1 omission per consultation under ideal audio conditions. (Further details on this will be presented in a future publication.) The positive correlation between increased microphone distance and the rise in errors (specifically omissions) highlights the importance of microphone placement, set-up and testing in clinical settings. An unexpected discovery was that more expensive microphones did not necessarily provide better performance than cheaper microphones; and in fact, cheaper microphones could actually outperform them. The data was clear, however, that the integrated laptop microphone tested was particularly poor. It is suggested that a possible reason for this is a combination of the low sensitivity of the microphone (which could have been tuned to work optimally at typical laptop user distances), combined with the innate directionally of that microphone. It should also be noted that that the audio quality from different microphones in different laptops is, of course, highly variable [ 26 ]. The impact of background noise on accuracy emphasises the need for AI algorithms capable of filtering out environmental disturbances. It is evident that some types of noise, such as a baby crying, have minimal effect on the clinical summary generated. However, vocalisations, such as toddler chatter, caused notable degradation of the clinical summary. As the transcript does not record the toddler’s words, it appears that the errors are due to the Clinical AI Scribe struggling to isolate different voices when the toddler spoke over either the patient or the clinician. Additionally, and of particular interest, the pink noise generated by heavy rain also caused a particularly large error rate, even at lower (and more realistic) volumes. Due to its acoustic profile, pink noise is often used for sound masking [ 27 ], which may have contributed to the Clinical AI Scribe’s difficulty in distinguishing the speech. Informational noise, in the form of either irrelevant casual conversation or patient-derived medical terms (for example, a speculative patient self-diagnosis) was treated comparatively well by the clinical AI scribe. Only a single hallucination was observed (with informational noise), though there were a number of omissions. As the source of the informational noise (medical or non-medical) did not have a statistically significant correlation with the number of omissions, it appears that (either through context, voice-identifying, or some other method) the clinical AI scribe is able to distinguish between the clinician and patient, and record their vocalisations with appropriate weighting. This is important because a clinical note should be weighted to a clinician’s thoughts, and not the patient’s [ 28 ]. Throughout the study, by far the most prevalent type of error were omissions. This demonstrates the conservative nature of this commercial Clinical AI Scribe, and suggests that the LLM has been designed, when presented with a difficult audio transcription, to err on the side of caution (producing omissions) rather than take a ‘best guess’ (which could produce hallucinations). It is generally accepted that that it is better to omit than to have incorrect information. Omissions observed included: Not recorded when the symptoms began. Not recording follow-up time. Not recording progression of symptoms. Not recording current medication. Hallucinations were not a frequent type of error, though did occur, particularly with acoustic noise. Their appearance appeared random and unpredictable, with no hallucination repeated. Hallucinations observed included: Describes patient having tried epoxy, instead of ibuprofen. Called rash bumpy when it was explicitly stated as not. Recorded pain, when there was none. Reports unrelated pain in unrelated place. With the exception of the Informational Noise section, it was interesting to note the presence of inclusions (accurate points of information present in the new summary, but not the original summary). It was anticipated that the addition of either type of noise would cause a decrease in the number of points of information, but instead, in several cases, the clinical AI scribe noted additional truthful points which it had previously ignored. For example, the addition of a conversation in which the clinician and patient are socially friends and discuss babysitting, mother-in-laws, and arranging a playdate caused the AI to accurately report that the patient is a smoker in the clinical summary (the smoking discussion was temporally far removed from the inserted casual conversation). Whilst this information was accurate (it was mentioned in the original audio), it was not reported in the original summary. The mechanism behind this remains unclear, though it demonstrates the LLMs unpredictable sensitivity to user input. Implications for Clinical Practice Given the rapidly increasing international uptake of AI-powered documentation tools in primary care (and elsewhere in clinical workflows), it is essential to ensure the accuracy and reliability of these systems. These findings underscore several key considerations for the integration of AI scribe technology into primary care settings. Firstly, microphone placement should be optimised within the consultation environment, with recommendations favouring a dedicated microphone positioned within one metre of the clinician, to reduce omission errors. In particular, integrated laptop microphones should be avoided. It is possible that lapel microphones could offer an improvement, although this could impact user acceptance. Additionally, an additional microphone located at an examination couch (which could be upwards of 2 metres away [ 29 ]) may be required to ensure all conversation is captured. Secondly, whilst the AI clinical scribe was generally able to accommodate low levels of background noise, interference from other vocalisations (such as toddlers) or pink noise (such as heavy rain) should be carefully minimised. This is especially important when the noise volume is equal or greater than the volume of the speech. Thirdly, when non-medically relevant social conversations are present in a consultation, additional care should be taken in confirming the generated summary, to ensure that no omissions have occurred. The issue of hallucinations and omissions in AI-generated clinical summaries reinforces the fact that (as required by all commercial providers) clinician oversight remains a necessary safeguard. While AI has the potential to streamline documentation [ 1 ], its current limitations indicate that all generated summaries should undergo manual review before being integrated into patient records. Clinicians must be trained to identify potential AI errors and to verify the accuracy of recorded information. It must be emphasised that user overconfidence in the accuracy of the AI-generated summary should be guarded against, as errors occur regularly enough to be problematic, but infrequently enough to risk complacency. And whilst the effect of such an event would be variable, it is quite possible that delays in diagnosis, impacts on treatment, or even a fatal consequence may occur. Future Directions for Research While this study provides significant insights into the challenges associated with Clinical AI Scribes, further research is needed to understand and enhance the robustness and usability of these systems. It is important to acknowledge that these tools are undergoing swift advancements, and considerable enhancements in performance are anticipated in the near future. One important area for future study is the development of AI models trained on datasets that incorporate a greater variety of real-world clinical noise conditions. There are currently very few publicly available databases which also include the audio file, e.g. [ 18 ]. Additionally, investigations into multimodal AI scribe approaches, which integrate audio, text, and contextual patient data, may yield improvements in accuracy and relevance. Another avenue of exploration is the impact of AI scribe on clinician workload and efficiency. While AI scribes offer the potential to reduce administrative burden, it remains to be seen whether their implementation leads to measurable improvements in workflow and patient outcomes [ 4 , 30 ]. Longitudinal studies assessing the long-term effectiveness of AI-assisted documentation, including its influence on consultation dynamics and medico-legal considerations, will be essential for guiding future AI adoption in healthcare. Conclusion AI-powered clinical scribes have great potential to improve clinical documentation and alleviate administrative burdens for healthcare professionals [ 1 , 2 ]. This study highlights the challenges associated with the use of Clinical AI scribes, particularly its sensitivity to microphone quality/positioning and environmental noise. The findings reinforce the importance of microphone placement, with close-proximity, dedicated microphones yielding the most reliable results. Increased distance and the use of integrated microphones were associated with higher omission rates, suggesting that optimised audio capture strategies should be a priority in clinical settings. Background noise, particularly comprehensible speech (for example toddler chatter) and heavy rain (pink noise), also significantly impacted accuracy. Whilst both non-medical and medical informational noise (for example a conversation about tennis or speculative patient self-diagnosis), did contribute to an increase in errors, they were far less pronounced than those caused by acoustic noise. Ultimately, clinician oversight remains a key factor in ensuring accuracy of both the summary and the EHR. While AI has the potential to streamline documentation, human validation is necessary to correct potential errors. Structured review workflows will be crucial to maintaining data integrity and supporting safe clinical decision-making. Contributions JMC, ST, JK, and RL conceived and designed the study. TCD, JL, KLR, and TC were involved in the acquisition of data. TCD, JL, TC, and RL analysed the data. TCD, TC, and JMC drafted the manuscript. All authors approved the final version of the manuscript. The corresponding author attests that all listed authors meet authorship criteria and that no others meeting the criteria have been omitted. Funding CoDE was awarded a contract by BNSSG ICB (Bristol, North Somerset, and South Gloucestershire Integrated Care Board) to supply a flexible and adaptable physical space in the South West region of England, capable of simulating the primary care environment, including, but not limited to, General Practice, pharmacy, dentistry, optometry and podiatry. Contract title: Procurement for the Provision of Primary Care Laboratory(s) to NHS England South West Region. Contract 11066090 - Service Contract - NHS Primary Care Laboratory. The funders helped shape the original research question, but had no role in the collection, analysis, interpretation of data, writing of the report, or decision to submit the article for publication. The researchers confirm their independence from the funding bodies. All authors had access to all data and assume full responsibility for the integrity of the data and the accuracy of its analysis. Competing Interests All authors have completed the ICMJE uniform disclosure form at http://www.icmje.org/disclosure-of-interest/ and declare: authors had financial support from Bristol, North Somerset, and South Gloucestershire Integrated Care Board (BNSSG ICB) for the submitted work; no financial relationships with any organisations that might have an interest in the submitted work in the previous three years; JMC provides guidance to the BNSSG ICB; no other relationships or activities that could appear to have influenced the submitted work. Ethical Approval Ethical approval for the work undertaken was granted by the University of the West of England Ethics committee – No: CHSS.24.05.199. Data Availability Statement Outputs from the commercial Clinical AI Scribe, along with transcripts of the original audio files, will be made available upon reasonable request to the corresponding author. Transparency Statement The corresponding author affirms that the manuscript is an honest, accurate, and transparent account of the study being reported; that no important aspects of the study have been omitted; and that any discrepancies from the study as planned have been explained. Acknowledgements The authors thank NHS England South West for the funds used to establish the CoDE at UWE Bristol, and the participants of the Citizens Reference Group for their contributions. References ↵ Chustecki M . Benefits and Risks of AI in Health Care: Narrative Review . Interact J Med Res . 2024 ; 13 : e53616 . doi: 10.2196/53616 OpenUrl CrossRef ↵ Al Kuwaiti A , Nazer K , Al-Reedy A , et al. A Review of the Role of Artificial Intelligence in Healthcare . J Pers Med . 2023 ; 13 . ↵ Rajpurkar P , Chen E , Banerjee O , et al. AI in health and medicine . Nat Med . 2022 ; 28 : 31 – 8 . doi: 10.1038/s41591-021-01614-0 OpenUrl CrossRef PubMed ↵ van Buchem MM , Kant IMJ , King L , et al. Impact of a Digital Scribe System on Clinical Documentation Time and Quality: Usability Study . JMIR AI . 2024 ; 3 : e60020 . doi: 10.2196/60020 OpenUrl CrossRef ↵ Gebauer S , Eckert C . Survey of US physicians’ attitudes and knowledge of AI . BMJ Evid Based Med . 2024 . ↵ Yu K-H , Beam AL , Kohane IS . Artificial intelligence in healthcare . Nat Biomed Eng . 2018 ; 2 : 719 – 31 . doi: 10.1038/s41551-018-0305-z OpenUrl CrossRef PubMed ↵ Li X , Zhang L , Yang J , et al. Role of Artificial Intelligence in Medical Image Analysis: A Review of Current Trends and Future Directions . J Med Biol Eng . 2024 ; 44 : 231 – 43 . OpenUrl ↵ Gomez-Cabello CA , Borna S , Pressman S , et al. Artificial-Intelligence-Based Clinical Decision Support Systems in Primary Care: A Scoping Review of Current Clinical Implementations . Eur J Investig Health Psychol Educ . 2024 ; 14 : 685 – 98 . doi: 10.3390/ejihpe14030045 OpenUrl CrossRef PubMed ↵ Schwieger A , Angst K , de Bardeci M , et al. Large language models can support generation of standardized discharge summaries – A retrospective study utilizing ChatGPT-4 and electronic health records . Int J Med Inform . 2024 ; 192 : 105654 . doi: 10.1016/j.ijmedinf.2024.105654 OpenUrl CrossRef PubMed ↵ Liu F , Yang B , You C , et al. Retrieve, Reason, and Refine: Generating Accurate and Faithful Patient Instructions . In: Koyejo S , Mohamed S , Agarwal A , et al. , eds. Advances in Neural Information Processing Systems . Curran Associates, Inc . 2022 : 18864 – 77 . ↵ Gebauer S. Benchmarking And Datasets For Ambient Clinical Documentation: A Scoping Review Of Existing Frameworks And Metrics For AI-Assisted Medical Note Generation . medRxiv , doi: 10.1101/2025.01.29.25320859 . 2025 . OpenUrl Abstract / FREE Full Text ↵ Mess SA , Mackey AJ , Yarowsky DE . Artificial Intelligence Scribe and Large Language Model Technology in Healthcare Documentation: Advantages, Limitations, and Recommendations . Plast Reconstr Surg Glob Open . 2025 ; 13 : e6450 . doi: 10.1097/GOX.0000000000006450 OpenUrl CrossRef ↵ Asan O , Young HN , Chewning B , et al. How physician electronic health record screen sharing affects patient and doctor non-verbal communication in primary care . Patient Educ Couns . 2015 ; 98 : 310 – 6 . doi: 10.1016/j.pec.2014.11.024 OpenUrl CrossRef PubMed ↵ Seth P , Carretas R , Rudzicz F . The Utility and Implications of Ambient Scribe in Primary Care (Preprint) . JMIR AI . Published Online First: 4 October 2024. doi: 10.2196/57673 OpenUrl CrossRef ↵ Wu Y , Wu M , Wang C , et al. Evaluating the Prevalence of Burnout Among Health Care Professionals Related to Electronic Health Record Use: Systematic Review and Meta-Analysis . JMIR Med Inform . 2024 ; 12 . ↵ Van Veen D , Van Uden C , Blankemeier L , et al. Adapted large language models can outperform medical experts in clinical text summarization . Nat Med . 2024 ; 30 : 1134 – 42 . doi: 10.1038/s41591-024-02855-5 OpenUrl CrossRef PubMed ↵ Kernberg A , Gold JA , Mohan V . Using ChatGPT-4 to Create Structured Medical Notes From Audio Recordings of Physician-Patient Encounters: Comparative Study . J Med Internet Res . 2024 ; 26 . doi: 10.2196/54419 OpenUrl CrossRef PubMed ↵ Yim W wai , Fu Y , Ben Abacha A , et al. Aci-bench: a Novel Ambient Clinical Intelligence Dataset for Benchmarking Automatic Visit Note Generation . Sci Data . 2023 ; 10 . doi: 10.1038/s41597-023-02487-3 OpenUrl CrossRef Pyne Y , Wong YM , Fang H , et al. Analysis of ‘One in a Million’ primary care consultation conversations using natural language processing . BMJ Health Care Inform . 2023 ; 30 . doi: 10.1136/bmjhci-2022-100659 OpenUrl Abstract / FREE Full Text Zhang Y , Merck D , Tsai E , et al. Optimizing the Factual Correctness of a Summary: A Study of Summarizing Radiology Reports . Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics . Stroudsburg, PA, USA : Association for Computational Linguistics 2020 : 5108 – 20 . ↵ Alsentzer E , Murphy J , Boag W , et al. Publicly Available Clinical BERT Embeddings . Proceedings of the 2nd Clinical Natural Language Processing Workshop . Stroudsburg, PA, USA : Association for Computational Linguistics 2019 : 72 – 8 . ↵ Tierney AA , Gayre G , Hoberman B , et al. Ambient Artificial Intelligence Scribes to Alleviate the Burden of Clinical Documentation . NEJM Catal . 2024 ; 5 . doi: 10.1056/cat.23.0404 OpenUrl CrossRef ↵ Crossley JGM , Howe A , Newble D , et al. Sheffield assessment instrument for letters (SAIL): Performance assessment using outpatient letters . Med Educ . 2001 ; 35 : 1115 – 24 . doi: 10.1046/j.1365-2923.2001.01065.x OpenUrl CrossRef PubMed Web of Science ↵ Balloch J , Sridharan S , Oldham G , et al. Use of an ambient artificial intelligence tool to improve quality of clinical documentation . Future Healthc J . 2024 ; 11 : 100157 . doi: 10.1016/j.fhj.2024.100157 OpenUrl CrossRef ↵ Quiroz JC , Laranjo L , Kocaballi AB , et al. Challenges of developing a digital scribe to reduce clinical documentation burden . NPJ Digit Med . 2019 ; 2 . doi: 10.1038/s41746-019-0190-1 OpenUrl CrossRef PubMed ↵ Penney J , Gibson A , Cox F , et al. A Comparison of Acoustic Correlates of Voice Quality Across Different Recording Devices: A Cautionary Tale . Interspeech 2021 .ISCA: ISCA 2021 : 1389 – 93 . ↵ Zhang Y , Ou D , Kang S . The effects of masking sound and signal-to-noise ratio on work performance in Chinese open-plan offices . Applied Acoustics . 2021 ; 172 : 107657 . doi: 10.1016/j.apacoust.2020.107657 OpenUrl CrossRef ↵ Kim MK , Rouphael C , McMichael J , et al. Challenges in and Opportunities for Electronic Health Record-Based Data Analysis and Interpretation . Gut Liver . 2024 ; 18 : 201 – 8 . doi: 10.5009/gnl230272 OpenUrl CrossRef ↵ Malkin J . Medical and Dental Space Planning . 4th ed . Hoboken, NJ : John Wiley & Sons Inc 2014 . ↵ Haberle T , Cleveland C , Snow GL , et al. The impact of nuance DAX ambient listening AI documentation: a cohort study . Journal of the American Medical Informatics Association . 2024 ; 31 : 975 – 9 . doi: 10.1093/jamia/ocae022 OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted March 24, 2025. Download PDF Supplementary Material Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries Thomas C. Draper , Jason Leake , Kathryn Lamb-Riddell , Timothy Cox , John McCormick , Stephen Trowell , Janice Kiely , Richard Luxton medRxiv 2025.03.24.25324398; doi: https://doi.org/10.1101/2025.03.24.25324398 Share This Article: Copy Citation Tools The Impact of Acoustic and Informational Noise on AI-Generated Clinical Summaries Thomas C. Draper , Jason Leake , Kathryn Lamb-Riddell , Timothy Cox , John McCormick , Stephen Trowell , Janice Kiely , Richard Luxton medRxiv 2025.03.24.25324398; doi: https://doi.org/10.1101/2025.03.24.25324398 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Health Systems and Quality Improvement Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (299) Cardiovascular Medicine (4425) Dentistry and Oral Medicine (443) Dermatology (382) Emergency Medicine (607) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1507) Epidemiology (15221) Forensic Medicine (30) Gastroenterology (1123) Genetic and Genomic Medicine (6588) Geriatric Medicine (667) Health Economics (997) Health Informatics (4524) Health Policy (1368) Health Systems and Quality Improvement (1612) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15910) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (145) Nephrology (667) Neurology (6588) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1143) Occupational and Environmental Health (956) Oncology (3331) Ophthalmology (970) Orthopedics (369) Otolaryngology (420) Pain Medicine (435) Palliative Medicine (129) Pathology (663) Pediatrics (1690) Pharmacology and Therapeutics (691) Primary Care Research (710) Psychiatry and Clinical Psychology (5440) Public and Global Health (9219) Radiology and Imaging (2195) Rehabilitation Medicine and Physical Therapy (1369) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (710) Sports Medicine (529) Surgery (710) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'9ffa7f745d0b0db4',t:'MTc3OTQzOTAzNQ=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();