Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study

Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study | medRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-P4HH5NV'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study View ORCID Profile Caitlin Silvestri , View ORCID Profile Viemma Nwigwe , View ORCID Profile Subhash Krishnamoorthy , View ORCID Profile Cary B. Aarons doi: https://doi.org/10.1101/2025.03.16.25323986 Caitlin Silvestri 1 Columbia University Irving Medical Center, Department of Surgery , New York NY 2 New York Presbyterian Hospital , New York NY MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Caitlin Silvestri For correspondence: cs4004{at}cumc.columbia.edu Viemma Nwigwe 1 Columbia University Irving Medical Center, Department of Surgery , New York NY 2 New York Presbyterian Hospital , New York NY MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Viemma Nwigwe Subhash Krishnamoorthy 1 Columbia University Irving Medical Center, Department of Surgery , New York NY MD Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Subhash Krishnamoorthy Cary B. Aarons 1 Columbia University Irving Medical Center, Department of Surgery , New York NY MD, MSEd Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Cary B. Aarons Abstract Full Text Info/History Metrics Data/Code Preview PDF Abstract Objective Residency applications rely on traditional letters of recommendation ( t LORs) from faculty or mentors to evaluate applicants. However, interpretation of t LORs can be limited by potential biases, overuse of hyperbolic language, and a lack of longitudinal contact. We aimed to assess whether incorporating peer letters of recommendation ( p LORs) would add a complementary perspective to the holistic review of an applicant’s attributes and potential. Design All applicants to a single, university-based general surgery residency program were invited to submit an optional p LOR in the 2023-24 recruitment cycle. Thematic analysis of applicants’ p LORs and t LORs was performed to identify patterns and sentiments. Setting Large general surgery residency program at a single, tertiary academic center. Participants Applicants selected for an interview for a general surgery residency program who submitted a p LOR in addition to their t LORs ( n =95). Results Ninety-five applicants (78%) selected for interview submitted a p LOR along with their standard application to the categorical ( n= 77) and preliminary ( n= 18) tracks. Peer letter writers knew applicants for an average of 6.14 years (SD 4.7). Thematic analysis identified notable differences in p LORs: 1) peer letter writers more often evaluated applicants across diverse settings (professional and personal) over longer time periods, 2) p LORs placed greater emphasis on the applicants’ impact on others (peers, individuals, patients), and 3) provided more specific, tangible examples of each positive attribute. Lastly, p LORs summative assessments often included personal language while t LORs tended to stratify applicants using percentiles or coded language. Conclusion Peer letters of recommendation offer a unique, complementary perspective in the holistic residency application review process. Compared with traditional letters, p LORs provide a richer context of an applicant’s impact in a community of their peers, more often providing tangible examples. This perspective is crucial for evaluating applicants as we build diverse and collaborative learning communities each year. Funding This research did not receive any specific grant from funding agencies in the public, commercial, or not-for-profit sectors. ACGME Core Competencies Professionalism (P), Interpersonal and Communication Skills (ICS) Highlights Peer letters of recommendation ( p LORs) offer a unique perspective in applicant review p LOR writers assess applicants across diverse settings over longer time periods p LORs emphasize applicants’ community impact with tangible, specific examples t LORs use ranking language, while p LORs include more personal, narrative assessments pLORs complement t LORs in holistic residency selection and evaluation processes Introduction When applying for general surgery residency, medical student applicants submit both subjective and objective data through the Electronic Residency Application Service (ERAS) 1 , including but not limited to a personal statement, curriculum vitae (CV) components, medical school transcripts, United States Medical Licensing Exam (USMLE) scores, and letters of recommendation (LORs) 2 . Historically, general surgery residencies review and place emphasis on an applicant’s academic metrics such as exam scores, clerkship grades, publications, research experience, and Alpha Omega Alpha (AOA) membership 3 – 5 . However, these quantitative measures of evaluation do not necessarily capture the full breadth of an applicant’s potential, and studies have demonstrated variability in their ability to predict success in residency 6 , 7 . Beyond academic metrics, residency selection committees also consider qualitative components such as personal statements and LORs. The role of the personal statement in the recruitment process remains inconsistent, with program directors differing in how they use it for interview selection and ranking 8 . Furthermore, studies have revealed substantial interreader variability, with a single personal statement often receiving multiple conflicting evaluations 9 , 10 . Similarly, LORs are a required element of the ERAS application and are intended to provide insight into an applicant’s abilities, accomplishments, and readiness for residency. Traditional letters of recommendation ( t LORs), typically authored by department chairs, faculty, or mentors, offer insights into an applicant’s performance based on personal interactions and peer comparisons. However, their reliability and impact in the selection process have been increasingly scrutinized due to concerns about bias, high interreader variability, and inconsistent influence on decision-making 11 – 14 . Additionally, LORs are frequently overly laudatory, contain hyperbolic language, and rely on coded phrasing that can obscure meaningful distinctions between applicants, further complicating their role in the selection process 15 , 16 . Recognizing the limitations of our current evaluation processes and metrics, the Association of American Medical Colleges (AAMC) developed a holistic review framework, which seeks to assess applicants as whole individuals rather than relying solely on academic achievements 17 . Holistic review allows for flexible, individualized, and mission-driven applicant review, aligning with the growing focus on improving diversity and equity in the physician workforce. Efforts to enhance holistic review have led to notable changes in the residency application process. One of the most significant shifts was the transition of the USMLE Step 1 to a pass/fail scoring system by the National Board of Medical Examiners (NBME), a move aimed at encouraging a broader evaluation of applicants 18 . This transition has created an opportunity for programs to redefine selection criteria and prioritize other factors, including interpersonal skills, leadership, and professionalism. As residency selection continues to evaluate candidates, LORs remain a key component of holistic review, despite their well-documented limitations. To provide a more comprehensive evaluation of applicants, new strategies should be explored to supplement t LORs. One potential tool is the inclusion of p LORs, which unlike t LORs, are authored by peers such as classmates, co-workers, friends, teammates, or members of shared organizations who can also offer unique perspectives on an applicant’s qualities. p LORs have been utilized in the undergraduate school admission process 19 , but have not yet been used in the residency recruitment. The introduction of p LORs in residency recruitment may help bridge the gaps posed by t LORs and other traditional application metrics by providing a different, complementary perspective of an applicant’s potential. The primary objective of this study is to evaluate the contextual differences between t LORs and p LORs for applicants to a single university-based general surgery residency program. We hypothesize that p LORs will offer a richer, complementary perspective compared to traditional faculty-authored letters, ultimately strengthening the holistic review process. Methods This study was reviewed and deemed exempt from full review by the Columbia University Institutional Review Board (IRB) under protocol number AAAU9012. In accordance with ethical research guidelines, participant confidentiality was maintained throughout data collection and analysis. No identifiable applicant information was included in the study. This study adheres to the Standards for Reporting Qualitative Research (SRQR) 20 to ensure transparency, rigor, and completeness in reporting. Participants A single, university-based general surgery residency program invited all applicants to submit an optional p LOR during the 2023-24 residency recruitment cycle. The study included applicants who received an interview invitation and submitted both an optional p LOR and at least one t LOR. No applicants were excluded based on demographics, specialty interests, or letter content. After the residency match process concluded, we retrospectively reviewed the data, ensuring that our analyses did not influence recruitment or selection outcomes. Data Collection The Electronic Residency Application Service (ERAS), managed by the Association of American Medical Colleges (AAMC) 1 , collects t LORs as part of the standard residency application process. Faculty members and department chairs typically write these letters. To ensure consistency, we selected a single t LOR per applicant, specifically the first t LOR written by a surgical faculty member within the ERAS application packet. Applicants submitted a single, optional, p LOR separately from their ERAS application. Each applicant identified a peer—such as a close friend, classmate, co-worker, teammate, research collaborator, or member of the same interest group or organization—to write a recommendation. Applicants were explicitly instructed that the letter could not be written by a family member, someone in a supervisory role, or a classmate applying into the same specialty. Peer letter writers were instructed to submit their letters directly to the residency program, with approximately 10 days to complete them from the time the application was downloaded by the program. When submitting the p LOR, letter writers also provided demographic information, including the nature of the relationship to the applicant and length of time known. Reflexivity Author [BLINDED-C], a male Professor of Surgery, serves as the Program Director (PD) of a general surgery residency program and Vice Chair of Education (VCE) in the Department of Surgery. He holds a medical degree and a Master’s of Science in Education (MSEd), where he received formal training in qualitative research methods. Before initiating this study, he participated in the residency application process as program director, overseeing the holistic review process for interview selection and conducting applicant interviews. His perspective provided a broader contextual understanding that informed the thematic analysis. Authors [BLINDED-A] and [BLINDED-B], female general surgery residents, collaborated with [BLINDED-C] in data coding and analysis. Both had received informal training in qualitative research methods and had prior experience in the residency application review process, contributing to holistic applicant evaluations for interview selection. Unlike [BLINDED-C], they did not conduct applicant interviews during the study cycle, minimizing potential bias in the generation and interpretation of themes. Analytic Approach Author [BLINDED-C] de-identified all LORs by removing identifiable information, including medical school names, applicant names, and AAMC ID numbers, before analysis. The authors used Taguette (version 1.4.1-60-ga73a5e2), a free qualitative analysis software, to organize coding and develop codebooks. Using an open, inductive coding approach, authors [BLINDED-C], [BLINDED-A], and [BLINDED-B] independently reviewed all p LORs and t LORs and developed separate codebooks based on emergent content. Throughout the coding process, the authors conducted debriefing sessions to refine initial codes and definitions, ensuring alignment and consistency. After finalizing codebooks, each author re-coded their assigned LORs to reflect the refined framework. Following individual coding, all authors collaboratively constructed themes through group discussions and consensus-building. Due to the de-identified nature of the data, we could not conduct member checking by inviting participants to review the identified themes. However, we ensured trustworthiness and rigor through investigator triangulation via independent coding by multiple authors, peer debriefing sessions, and consensus-based construction of final themes 21 . Results Participant Demographics During the 2023-2024 recruitment period, the program received approximately 1600 applications to the general surgery residency, including both categorical and preliminary tracks. Of these, 122 applicants were selected for interviews, including 83 (68%) for the categorical track and 39 (32%) for the preliminary track. Among interviewees from both tracks, 95 (78%) submitted an optional p LOR along with their standard ERAS application. Of the 95 applicants included in the analysis, 77 (81%) were categorical and 18 (19%) were preliminary track applicants ( Table 1 ). View this table: View inline View popup Download powerpoint Table 1. Applicant demographics of the general interview pool vs. p LOR submission pool pLOR Writer Demographics Among the 95 submitted p LORs, the writers had known the applicants for an average of 6.14 years (SD 4.7, range 0.2–23 years). The primary relationship between the letter writer and the applicant varied, with the most common being friends ( n =55, 58%) and classmates ( n =32, 34%). Less common relationships included teammates ( n =3, 3%), research collaborators ( n =2, 2%), co-workers (n=2, 2%), and coaches ( n =1, 1%) ( Table 2 ). Additionally, 15 (16%) of p LOR writers indicated they had multiple relationships with the applicant. View this table: View inline View popup Download powerpoint Table 2. Primary relationship of p LOR writers to the applicant Overarching Themes We compared p LOR and t LOR submissions and identified four major themes ( Table 3 ). View this table: View inline View popup Table 3. Thematic analysis of p LORs and t LORs Qualifications Both p LOR and t LOR writers commonly discussed their qualifications for writing the LOR, but they differed in the type of content used to establish their credibility to the reader. t LOR writers typically emphasized their professional and academic roles, citing institutional appointments and leadership positions such as medical student clerkship director, department chair, or program director to support their ability to assess the applicant and compare to other students: “ As director of our student education programs and of the Surgical Education Fellowship, I have had close and direct involvement in observing, assessing and interacting with our students” (P14) . In contrast, p LOR writers more frequently established their qualifications by referencing the length of time and the variety of contexts in which they knew the applicant, positioning their personal experiences as foundations for their assessment of the applicant: I have known [APPLICANT] for three years as a classmate and close friend. In this capacity, I can comment on his personal characteristics as well as academic engagements (P15) . Positive attributes Both p LOR and t LOR writers consistently highlighted the applicant’s positive attributes throughout their letters, but they differed in the types of attributes emphasized and the ways in which they substantiated these qualities. In t LORs, writers most commonly described attributes related to clinical performance, technical skills, and foundational medical knowledge. “He seemed to thrive in that environment and impressed us all with his clinical, technical acumen, his rapid progress, and his extraordinary work ethic and learning efforts” (P14) . These letters frequently incorporated input from multiple evaluators, often through composite letters, feedback from additional evaluators, or imported quotations from summative assessments: “In preparation of this letter, I sought the opinion of the chief resident who worked with her as a sub-intern, which generated the following comments: “[APPLICANT] learns quickly, has a solid fund of knowledge and reasons well from first principles. She has a good attitude and is always willing to help the team. She routinely came in early and stayed late to make sure her patients were given optimal care. She was very helpful with floor work and quickly learned how to do routine patient care tasks. Her technical skills in the OR were very good for her level, certainly more advanced than I would expect for a medical student. She is very self-aware of any deficiencies and is always working hard to improve . ” (P54) . In contrast, p LOR writers more frequently emphasized interpersonal relationships and character traits, with significantly less focus on clinical or technical abilities. Furthermore, p LOR writers tended to support these attributes with detailed, anecdotal examples illustrating the applicant’s behavior in specific contexts: “One specific example was during a group project we worked on together in our neurobiology class. Our task was to analyze a complex case study and present our findings to the class. [APPLICANT] took charge of coordinating the project and immediately led a weekly discussion group with other students. [APPLICANT] also encouraged open communication and brainstorming, creating a safe environment where everyone felt comfortable sharing their thoughts and suggestions” (P58 ). Roles and accomplishments Both t LOR and p LOR writers consistently highlighted the applicant’s roles and accomplishments, but they differed in the types of achievements they emphasized. t LOR writers primarily focused on accomplishments typically found in the applicant’s CV, such as awards, grades, USMLE scores, leadership positions, research achievements, and scholarly contributions (e.g., publications, abstracts, and oral presentations). These letters often framed the applicant’s accomplishments within a structured, academic context: “After coming to [Institution], his scholarly activity continued to be outstanding, with Alpha Omega Alpha (AOA) recognition, many Honors grades, a notable USMLE Step 2 CK score of 275, and numerous research activities” (P24) . In contrast, p LOR writers tended to discuss roles and accomplishments within the context of the applicant’s impact on their community—whether among peers, in research settings, or with patients. These letters highlighted the applicant’s influence on individuals and groups, often illustrating their mentorship, leadership, and support for others: “[APPLICANT] is well known among her peers as a go-to person for many medical school-related concerns. I have personally witnessed [APPLICANT] mentor and guide many students through the early years of their medical training. Peers regularly refer students to [APPLICANT] when they need a listening ear, resources, or advice navigating their way through all steps of the medical school journey” (P8 ). Summative assessment Both t LOR and p LOR writers included a summative assessment in the letter, but differed in how they characterized the applicant’s overall impact. t LOR writers frequently used stratification methods, such as percentiles or coded language (e.g., outstanding, excellent, exceptional), to rank applicants relative to their peers. Additionally, they often indicated whether their institution would actively recruit the applicant to their program: “In summary, [APPLICANT] is easily within the top 1% of medical students I’ve had the honor of training … I will be heavily recruiting him for our own program” (P16 ). In contrast, p LOR writers tended to offer summative assessments that were more personal, often expressing a desire to work with the applicant as a future colleague or entrust them with the care of loved ones and family members: “[APPLICANT] is going to be a phenomenal co-resident, and I truly envy his future colleagues. He is authentically kind, a team player, a hard worker, and a mentor to the learners around him. [APPLICANT] is the kind of person I want treating my family members in the future. I give him my highest recommendation possible” (P16 ). Discussion This qualitative analysis compared traditional letters of recommendation ( t LORs) and peer letters of recommendation ( p LORs) for applicants interviewing at a single general surgery residency program. To our knowledge, our study is the first to incorporate p LORs into the residency recruitment process and qualitatively compare t LORs and p LORs. We identified four key themes illustrating how t LORs and p LORs capture complementary perspectives on applicants, albeit through different lenses. Specifically, t LORs tend to emphasize academic achievements, technical skills, and summative comparisons by employing coded language, percentiles, and a desire to recruit them. Compared to p LORs which provide a more personal appraisal by highlighting an applicant’s character, interpersonal relationships, engagement within the community, and the direct impact they have on peers and patients, all while providing more detailed examples of these traits. Traditional LORs are a longstanding component of residency applications, but come with notable limitations. Up to 20% of t LORs are written by department chairs - close to half of whom have limited direct clinical experience with the applicant - or are written from faculty with single, narrow contact with the applicant 22 . Prior research has also shown that nearly 17% of t LORs lack evidence of direct clinical supervision in addition to being overly laudatory, and often including hyperbolic, coded, and biased language 22 , 11 , 23 . While key components of an applicants’ CV may become diluted within the ERAS application, t LOR writers frequently recapitulate what they find to be most important aspects including exam scores, clinical grades, awards, and research productivity. Lastly, we found that t LOR writers often employ summative language, including percentiles and coded language and frequently include whether the applicant will be recruited to their program - a factor that previous studies have identified as highly impactful for faculty in selection 24 . Despite their limitations, t LORs can still provide critical components within the residency recruitment process, offering programs valuable insights into attributes deemed essential for success in surgical training. In an effort to address the various limitations of t LORs, some specialties have adopted standardized letters of recommendations (SLORs), which are intended to offer a more uniform, nationalized scale for evaluating applicants 25 . Surgical specialties have tried to implement SLORs, however despite their aim for consistency and applicant stratification, are still found to be overly favorable to applicants 25 – 28 . A SLOR was created for general surgery by the American College of Surgeons (ACS) and Association for Program Directors in Surgery (APDS) 29 , however, widespread adoption and utilization has not occurred. Focus groups with key stakeholders, including general surgery clerkship and program directors, have expressed reservations about SLORs due to performance inflation and reliance on scaled assessments of attributes without providing examples 23 . This further emphasizes the need for alternative approaches. Our study demonstrates that p LORs can address some of these limitations by providing a complementary perspective of applicants. Unlike t LORs, p LOR writers generally have longer and more varied relationships with the applicants. Additionally, they focus on more personal attributes - such as character, interpersonal relationships, and community engagement - that are typically not captured in t LORs. p LORs also offer specific examples and narratives about an applicant’s impact on peers, their community, and patients - a context that is less common to read about it t LORs, but critical as we endeavor to create effective and diverse learning teams each year. Lastly, their summative assessment is a more personal reflection, discussing genuine desires to work with the applicant or having them care for a family member, rather than t LORs using percentiles and coded language to compare to their peers. This personal lens is particularly important in an era when objective data within residency recruitment is declining, as it provides program directors with a more nuanced and holistic review of an applicant’s potential. This study has several key limitations. First, it was conducted at a single academic center and only evaluated LORs from applicants that had been selected for interviews, which may limit the generalizability. Moreover, the optional nature of the p LOR submissions could have introduced selection bias, with applicants who were able to secure a supportive peer recommendation might not represent the entire applicant pool, potentially skewing the overall portrayal more positively. Although our sample size was sufficient to reach thematic saturation, further research with larger and more diverse cohorts is necessary 30 . Given these findings, we recommend that residency program directors consider incorporating p LORs as an optional component in the ERAS application. Such integration would allow programs that value perspectives on interpersonal relationships, communication, community impact, and applicants’ character to enhance their holistic review process. Future research should involve multi-institutional studies to validate our findings and explore the utility of p LORs in other specialties. Conclusion In conclusion, our study demonstrates that p LORs provide valuable, complementary insights to traditional evaluations by emphasizing personal attributes and community impact that are often overlooked. Despite limitations such as single-center data and potential selection bias, these findings support the integration of p LORs into the residency recruitment process to enhance holistic review. Future multi-institutional studies will be essential to further validate and refine the use of p LORs in applicant assessment. Data Availability All data produced in the present study are available upon reasonable request to the authors Abbreviations p LORs peer letters of recommendation t LORs traditional letters of recommendation References 1. ↵ Applying to Residencies with the ERAS® System . Students & Residents . Accessed February 17, 2025 . https://students-residents.aamc.org/applying-residencies-eras/applying-residencies-eras-system 2. ↵ What You Need to Know About the 2025 ERAS® Application Season. Students & Residents . Accessed February 18, 2025 . https://students-residents.aamc.org/applying-residencies-eras/what-you-need-know-about-2025-eras-application-season 3. ↵ Hebert JC . Characteristics of highly ranked applicants to general surgery residency programs: are we assessing the right criteria?: Comment on “Characteristics of highly ranked applicants to general surgery residency programs.” JAMA Surg . 2013 ; 148 ( 5 ): 418 . OpenUrl PubMed 4. Collins RA , Nimmer K , Sheriff SA , et al. Characteristics Associated with Successful Residency Match in General Surgery . Ann Surg Open . 2024 ; 5 ( 3 ): e469 . OpenUrl 5. ↵ Dort JM , Trickey AW , Kallies KJ , Joshi ART , Sidwell RA , Jarman BT . Applicant Characteristics Associated With Selection for Ranking at Independent Surgery Residency Programs . J Surg Educ . 2015 ; 72 ( 6 ): e123 – e129 . OpenUrl PubMed 6. ↵ The relationship between criteria used to select residents and performance during residency . The American Journal of Surgery . 1997 ; 173 ( 4 ): 326 – 329 . OpenUrl PubMed 7. ↵ Longo WE . The Predictive Value of General Surgery Application Data for Future Resident Performance . Journal of Surgical Education . 2011 ; 68 ( 6 ): 513 – 518 . OpenUrl PubMed 8. ↵ National Resident Matching Program, Data Release and Research Committee . Results of the 2021 NRMP Program Director Survey .; 2021 . 9. ↵ White BAA , Sadoski M , Thomas S , Shabahang M. Is the evaluation of the personal statement a reliable component of the general surgery residency application? J Surg Educ . 2012 ; 69 ( 3 ): 340 – 343 . OpenUrl CrossRef PubMed 10. ↵ Surgical Residency Recruitment—Opportunities for Improvement . Journal of Surgical Education . 2016 ; 73 ( 6 ): e104 – e110 . OpenUrl PubMed 11. ↵ Turrentine FE , Dreisbach CN , St Ivany AR , Hanks JB , Schroen AT . Influence of gender on surgical residency applicants’ recommendation letters . J Am Coll Surg . 2019 ; 228 ( 4 ): 356 - 365.e3 . OpenUrl PubMed 12. Dirschl DR , Adams GL . Reliability in Evaluating Letters of Recommendation . Academic Medicine . 2000 ; 75 ( 10 ): 1029 . OpenUrl CrossRef PubMed Web of Science 13. Khan S , Kirubarajan A , Shamsheri T , Clayton A , Mehta G. Gender bias in reference letters for residency and academic medicine: a systematic review . Postgrad Med J . 2023 ; 99 ( 1170 ): 272 – 278 . OpenUrl PubMed 14. ↵ Rojek AE , Khanna R , Yim JWL , et al. Differences in Narrative Language in Evaluations of Medical Students by Gender and Under-represented Minority Status . J Gen Intern Med . 2019 ; 34 ( 5 ): 684 – 691 . OpenUrl PubMed 15. ↵ Go C , Sachdev U. Letters of recommendation: Nuanced bias or useful affirmation? J Vasc Surg . 2021 ; 74 ( 2S ): 29S – 32S . OpenUrl PubMed 16. ↵ Saudek K , Saudek D , Treat R , Bartz P , Weigert R , Weisgerber M. Dear Program Director: Deciphering Letters of Recommendation . J Grad Med Educ . 2018 ; 10 ( 3 ): 261 – 266 . OpenUrl PubMed 17. ↵ Website. Holistic Review . AAMC . Accessed February 18, 2025 . https://www.aamc.org/services/member-capacity-building/holistic-review 18. ↵ Lin GL , Nwora C , Warton L. Pass/fail score reporting for USMLE Step 1: An opportunity to redefine the transition to residency together . Acad Med . 2020 ; 95 ( 9 ): 1308 – 1311 . OpenUrl PubMed 19. ↵ Peer Recommendation . Dartmouth Admissions . January 4, 2018 . Accessed February 18, 2025 . https://admissions.dartmouth.edu/glossary-term/peer-recommendation 20. ↵ O’Brien BC , Harris IB , Beckman TJ , Reed DA , Cook DA . Standards for reporting qualitative research: a synthesis of recommendations . Acad Med . 2014 ; 89 ( 9 ): 1245 – 1251 . OpenUrl CrossRef PubMed 21. ↵ Nowell LS , Norris JM , White DE , Moules NJ . Thematic analysis . Int J Qual Methods . 2017 ; 16 ( 1 ): 160940691773384 . OpenUrl 22. ↵ The content and value of letters of recommendation in the resident candidate evaluative process . Current Surgery . 2002 ; 59 ( 1 ): 79 – 83 . OpenUrl CrossRef PubMed 23. ↵ Naples R , French JC , Lipman JM . Best Practices in Letters of Recommendation for General Surgery Residency: Results of Expert Stakeholder Focus Groups . J Surg Educ . 2020 ; 77 ( 6 ): e121 – e131 . OpenUrl PubMed 24. ↵ What Are We REALLY Looking for in a Letter of Recommendation? Journal of Surgical Education . 2019 ; 76 ( 6 ): e118 – e124 . OpenUrl PubMed 25. ↵ Jackson JS , Bond M , Love JN , Hegarty C. Emergency Medicine Standardized Letter of Evaluation (SLOE): Findings From the New Electronic SLOE Format . J Grad Med Educ . 2019 ; 11 ( 2 ): 182 – 186 . OpenUrl PubMed 26. Hu AC , Gu JT , Wong BJF. Objective measures and the standardized letter of recommendation in the otolaryngology residency match . Laryngoscope . 2020 ; 130 ( 3 ): 603 – 608 . OpenUrl PubMed 27. Kang HP , Robertson DM , Levine WN , Lieberman JR . Evaluating the Standardized Letter of Recommendation Form in Applicants to Orthopaedic Surgery Residency . J Am Acad Orthop Surg . 2020 ; 28 ( 19 ): 814 – 822 . OpenUrl PubMed 28. ↵ Samade R , Balch Samora J , Scharschmidt TJ , Goyal KS . Use of Standardized Letters of Recommendation for Orthopaedic Surgery Residency Applications: A Single-Institution Retrospective Review . J Bone Joint Surg Am . 2020 ; 102 ( 4 ): e14 . OpenUrl PubMed 29. ↵ Resources for Governors . ACS . Accessed February 22, 2025 . https://www.facs.org/about-acs/governance/board-of-governors/resources/ 30. ↵ Hennink M , Kaiser BN . Sample sizes for saturation in qualitative research: A systematic review of empirical tests . Soc Sci Med . 2022 ; 292 : 114523 . OpenUrl CrossRef PubMed View the discussion thread. Back to top Previous Next Posted March 18, 2025. Download PDF Data/Code Email Thank you for your interest in spreading the word about medRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study Message Subject (Your Name) has forwarded a page to you from medRxiv Message Body (Your Name) thought you would like to see this page from the medRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study Caitlin Silvestri , Viemma Nwigwe , Subhash Krishnamoorthy , Cary B. Aarons medRxiv 2025.03.16.25323986; doi: https://doi.org/10.1101/2025.03.16.25323986 Share This Article: Copy Citation Tools Exploring the Value of Peer Letters of Recommendation in the Holistic Review of Surgery Residency Applications: A Qualitative Study Caitlin Silvestri , Viemma Nwigwe , Subhash Krishnamoorthy , Cary B. Aarons medRxiv 2025.03.16.25323986; doi: https://doi.org/10.1101/2025.03.16.25323986 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Surgery Subject Areas All Articles Addiction Medicine (568) Allergy and Immunology (863) Anesthesia (300) Cardiovascular Medicine (4435) Dentistry and Oral Medicine (444) Dermatology (382) Emergency Medicine (608) Endocrinology (including Diabetes Mellitus and Metabolic Disease) (1509) Epidemiology (15227) Forensic Medicine (30) Gastroenterology (1124) Genetic and Genomic Medicine (6597) Geriatric Medicine (668) Health Economics (997) Health Informatics (4534) Health Policy (1368) Health Systems and Quality Improvement (1613) Hematology (540) HIV/AIDS (1264) Infectious Diseases (except HIV/AIDS) (15916) Intensive Care and Critical Care Medicine (1103) Medical Education (623) Medical Ethics (146) Nephrology (667) Neurology (6599) Nursing (346) Nutrition (998) Obstetrics and Gynecology (1144) Occupational and Environmental Health (957) Oncology (3332) Ophthalmology (974) Orthopedics (369) Otolaryngology (420) Pain Medicine (436) Palliative Medicine (130) Pathology (663) Pediatrics (1693) Pharmacology and Therapeutics (691) Primary Care Research (711) Psychiatry and Clinical Psychology (5447) Public and Global Health (9230) Radiology and Imaging (2198) Rehabilitation Medicine and Physical Therapy (1370) Respiratory Medicine (1196) Rheumatology (593) Sexual and Reproductive Health (712) Sports Medicine (530) Surgery (712) Toxicology (99) Transplantation (289) Urology (265) (function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a002c3bb9efd58f4',t:'MTc3OTUyNTcxOA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();