Parsing GTF and FASTA files using the eccLib Library

Parsing GTF and FASTA files using the eccLib Library | bioRxiv /* */ /* */ <!-- <!-- /*! * yepnope1.5.4 * (c) WTFPL, GPLv2 */ (function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f<d;f++)g=a[f].split("="),(e=z[g.shift()])&&(c=e(c,g));for(f=0;f<b;f++)c=x[f](c);return c}function g(a,e,f,g,h){var i=b(a),j=i.autoCallback;i.url.split(".").pop().split("?").shift(),i.bypass||(e&&(e=d(e)?e:e[a]||e[g]||e[a.split("/").pop().split("?")[0]]),i.instead?i.instead(a,e,f,g,h):(y[i.url]?i.noexec=!0:y[i.url]=1,f.load(i.url,i.forceCSS||!i.forceJS&&"css"==i.url.split(".").pop().split("?").shift()?"c":c,i.noexec,i.attrs,i.timeout),(d(e)||d(j))&&f.load(function(){k(),e&&e(i.origUrl,h,g),j&&j(i.origUrl,h,g),y[i.url]=2})))}function h(a,b){function c(a,c){if(a){if(e(a))c||(j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}),g(a,j,b,0,h);else if(Object(a)===a)for(n in m=function(){var b=0,c;for(c in a)a.hasOwnProperty(c)&&b++;return b}(),a)a.hasOwnProperty(n)&&(!c&&!--m&&(d(j)?j=function(){var a=[].slice.call(arguments);k.apply(this,a),l()}:j[n]=function(a){return function(){var b=[].slice.call(arguments);a&&a.apply(this,b),l()}}(k[n])),g(a[n],j,b,n,h))}else!c&&l()}var h=!!a.test,i=a.load||a.both,j=a.callback||f,k=j,l=a.complete||f,m,n;c(h?a.yep:a.nope,!!i),i&&c(i)}var i,j,l=this.yepnope.loader;if(e(a))g(a,0,l,0);else if(w(a))for(i=0;i (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;j.type='text/javascript';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-M677548'); Skip to main content Home About Submit ALERTS / RSS Search for this keyword Advanced Search New Results Parsing GTF and FASTA files using the eccLib Library View ORCID Profile Tomasz Chady , View ORCID Profile Zuzanna Karolina Filutowska doi: https://doi.org/10.1101/2025.04.24.650481 Tomasz Chady 1 Faculty of Mathematics and Computer Science, Adam Mickiewicz University, ul, Uniwersytetu Poznańskiego 4 , 61-614, Poznań, Poland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Tomasz Chady Zuzanna Karolina Filutowska 2 Faculty of Biology, Adam Mickiewicz University, ul, Uniwersytetu Poznańskiego 6 , 61-614, Poznań, Poland Find this author on Google Scholar Find this author on PubMed Search for this author on this site ORCID record for Zuzanna Karolina Filutowska For correspondence: platyna{at}amu.edu.pl Abstract Full Text Info/History Metrics Preview PDF Abstract Summary Leveraging the Python/C API, eccLib was developed as a high-performance library designed for parsing genomic files and analysing genomic contexts. To the best of the authors’ knowledge, it is the fastest Python-based solution available. With eccLib, users can efficiently parse GTF/GFFv3 and FASTA files and utilise the provided methods for additional analysis. Availability and Implementation This library is implemented in C and distributed under the GPL-3.0 licence. It is compatible with any system that has the Python interpreter (CPython) installed. The use of C enables numerous optimisations at both the implementation and algorithmic levels, which are either unachievable or impractical in Python. Contact tomcha{at}st.amu.edu.pl , platyna{at}amu.edu.pl , eccdna{at}eccdna.pl Supplementary information This library is available for installation from the Python Package Index (PyPI) under the name eccLib https://pypi.org/project/eccLib/ . The source code is available at https://gitlab.platinum.edu.pl/eccdna/eccLib . More detailed documentation can be accessed at https://gitlab-pages.platinum.edu.pl/eccdna/eccLib/ . Introduction GTF/GFF version 2 and FASTA file formats are widely used in genomics owing to their simplicity and ease of use. Consequently, parsing these files is often the first step in many projects, and can become a bottleneck. Indeed, this was the experience of the authors, who found a Python-based parser to be too slow and memory-intensive for their requirements. To address this issue, numerous libraries exist to facilitate this process. The gffutils module ( Dale, 2011 ) parses GTF files into an SQLite database. The gtfparse module ( Openvax, 2015 ), the Biopython-affiliated bcbio-gff module ( chapmanb, 2013 ), and the GTF parser within the PyRanges library ( Stovner and Sætrom, 2019 ) all return representations of GTF files. For FASTA files, the Biopython library includes a parser called SeqIO , and the standalone library fastaparser ( Kronopt, 2020 ) is also capable of parsing FASTA files. However, the performance of some of these libraries may be suboptimal. To overcome these limitations, eccLib , a performance-oriented library written in C for Python was developed. The primary objective of eccLib is to accelerate the parsing of GTF and FASTA files in Python while providing efficient data structures for genomic data. Users can expect significant performance improvements, particularly with large datasets. Methods The standard implementation of the Python interpreter, CPython, provides an interface for integrating C code, known as the Python/C API. Using this API, developers can write C code that, once compiled and linked to the Python interpreter, can be invoked directly from Python code. This is exactly the interface utilised by numpy ( Harris et al., 2020 ) to deliver excellent performance for numerical computations. Development of eccLib was not without its challenges owing to its low-level nature. Debugging was particularly problematic when C memory errors propagated into Python reference counting errors. Despite this, the performance gains – even from a direct port of existing Python code into C – can be significant. However, the true performance improvements arise from low-level access to memory, which permits optimisations that are either unachievable or impractical in Python. Within eccLib , this low-level access was utilised to optimise the parsing of GTF and FASTA files, primarily by reducing the number of memory allocations to a minimum and avoiding unnecessary data copying, which is a common issue in high-level languages such as Python. Data Types In order to gain access to the internals of data structures so that they may be optimised, multiple custom data types were created. The GtfDict class is a Mapping object used to store data from GTF files. The core GTF entry fields are stored as separate members within the object, permitting access without a hash map lookup. For storing GTF attributes, a custom C hash map is employed, which is faster than a Python dict . This hash map, from the hashmap.h library ( Warden, 2010 ), utilises the xxHash hashing function ( Collet, 2012 ), thereby improving overall performance. Additionally, the GtfDict class includes several methods for basic data analysis. The FastaBuff class is used to store FASTA DNA data. The set I of IUPAC nucleotide codes ( Cornish-Bowden, 1985 ), can be mapped to a power set of DNA bases using f : I → 𝒫 ({ A, C, T, G }). This power set can then be mapped to a 4-bit integer, where each bit represents the presence of a specific base, using g : 𝒫 ({ A, C, T, G }) → {0, 1} 4 . The composition of these two functions allows for the efficient storage of sequences as compact binary data, specifically as unsigned 4-bit integers. In this implementation, these 4-bit integers are packed into bytes, enabling a memory usage reduction of at least 50% compared to Python str objects, with minimal performance overhead. For instance, the sequence “CG” may be represented as (01000010) 2 , and the sequence “DNT” as (10111111 00010000) 2 . For non-nucleotide characters, the Python str object is used as a fallback. GTF parsing The principal challenge in GTF parsing lies in string tokenisation and the optimal storage of attributes. To address this, a custom C tokeniser that does not modify the input data is utilised – thereby avoiding additional memory allocations – and a third-party hash map library is employed for caching and attribute storage. Caches are utilised for storing already encountered keys and values during parsing, thus avoiding redundant memory allocations for identical strings. In addition, the process of resolving URL and UTF-8 encodings was optimised by using a custom C function that simultaneously resolves both encodings, which significantly increases overall performance. FASTA parsing Parsing the FASTA format is considerably simpler than GTF processing. The primary challenge concerns the memory storage of sequences. To address this, the FastaBuff class is employed as the storage data type by the primary FASTA parser, parseFASTA() . This approach reduces the memory usage of the FASTA parser compared with implementations that use Python bytes or str objects. Results To assess the performance of the eccLib library, a benchmark was conducted. This benchmark was performed on a Linux system equipped with an Intel i9-9900KF CPU, 48.0 GB of RAM, and an SSD. The objective was to measure how quickly each parser could return a representation of the provided file. For iterative parsers, the outputs were appended to a list, thereby simulating the behaviour of a non-iterative parser. In turn, 20 new Python instances were launched for each parser, measuring the time with the datetime module and memory usage with the GNU Time utility. To better illustrate overall performance, the Time-Memory Product (TMP) was calculated using the formula TMP = Time × Memory, in appropriate units. GTF parsing benchmark Benchmark results suggest that the GTF parsing methods in eccLib significantly outperform both gtfparse and the parser within the PyRanges library. The benchmark file used for speed tests was the 1.4 GB Homo sapiens.GRCh38.108.gtf from Ensembl ( Dyer et al., 2024 ), chosen for its size and accessibility. Parsing methods within eccLib were compared against one another, the gtfparse read gtf function, and the PyRanges read gtf function. The results are presented in fig. 1 . Download figure Open in new tab Fig. 1. Speed and peak memory usage of different GTF parsing methods. The methods presented here, parseGTF and GtfFile , achieved TMP scores of 137 and 110, respectively. Third-party methods, gtfparse.read_gtf and pyranges.read_gtf , achieved TMP scores of 490 and 287, respectively. The results indicate that methods presented here are significantly faster at parsing GTF files compared to the alternatives, albeit with higher memory usage. eccLib.GtfFile parser is 3.9 times faster than pyranges.read_gtf , while consuming 1.5 times more memory. Contrary to expectations, the iterative parser was not significantly slower than the non-iterative parser. The authors attribute this to the hardware characteristics, which mitigated the performance penalty incurred by additional I/O operations. One factor that may have influenced the results is the high number of repeated GTF attribute keys and values in the benchmark file; however, the authors do not believe this had a significant impact on the overall outcomes. Unfortunately, gffutils and bcbio-gff could not be tested due to their poor performance on the test file. FASTA benchmark Testing indicates that the parseFASTA() function in eccLib outperforms contemporary FASTA parsers. The benchmarking was conducted using the same methodology as the GTF parsing benchmark. The test file used was the 3.2 GB Homo sapiens.GRCh38.dna.primary assembly.fa from Ensembl. The averaged results are shown in fig. 2 . Download figure Open in new tab Fig. 2. Speed and peak memory usage of different genomic FASTA parsing methods. The function presented here, parseFASTA() , achieved a TMP of 40, while fastaparser and SeqIO achieved TMP scores of 115 and 44, respectively. While parseFASTA() demonstrates a clear speed advantage, it incurs increased memory consumption. The fastaparser reader was also used in “quick” mode to manage memory usage. Since eccLib handles DNA sequences differently from protein sequences, a separate benchmark was conducted using the 85.1 MB Homo_sapiens.GRCh38.pep.all.fa from Ensembl. The results are illustrated in fig. 3 . Download figure Open in new tab Fig. 3. Speed and peak memory usage of different protein FASTA parsing methods. fastaparser.Reader achieved the best TMP of 67, with eccLib and SeqIO achieving TMP scores of 79 and 128, respectively. Here, the relative performance of eccLib is notably poorer; however, the speed advantage remains; the increased memory usage is largely attributable to the fact that protein sequences are not stored as efficiently as genomic sequences. Discussion The eccLib library provides a high-performance solution for parsing GTF and FASTA files in Python, thereby significantly enhancing project performance. Although it utilises more memory than alternative approaches, this is a consequence of its adoption of hash maps rather than fixed-field tables, which better accommodate the flexibility of GTF/GFF files. Despite the higher memory usage, the performance benefits are substantial, and the authors contend that these advantages justify the increased memory consumption. Future work will focus on expanding features and improving interoperability with other libraries. The choice of C as the implementation language was driven by the need for performance. However, the authors acknowledge the inherent issues associated with programming in C. The authors find the possibility of developing a library for Python with a similar feature set in a higher-level language, such as Rust, to be an interesting avenue for future exploration. Acknowledgements This work was supported by a grant from the Polish Ministry of Education and Science (MEIN) [0054/DIA/2014/43]. The authors declare that they have no competing interests. No new data were generated or analysed in support of this research. Data used for benchmarking were obtained from Ensembl ( Dyer et al., 2024 ). References ↵ chapmanb ( 2013 ). bcbio-gff python module . https://github.com/chapmanb/bcbb/tree/master/gff . ↵ Collet , Y. ( 2012 ). xxhash . https://github.com/Cyan4973/xxHash . ↵ Cornish-Bowden , A. ( 1985 ). Nomenclature for incompletely specified bases in nucleic acid sequences: recommendations 1984 . Nucleic acids research , 13 ( 9 ): 3021 . OpenUrl CrossRef PubMed Web of Science ↵ Dale , R. ( 2011 ). gffutils python module . https://github.com/daler/gffutils . ↵ Dyer , S. C. , Austine-Orimoloye , O. , Azov , A. G. , Barba , M. , Barnes , I. , Barrera-Enriquez , V. P. , Becker , A. , Bennett , R. , Beracochea , M. , Berry , A. , Bhai , J. , Bhurji , S. K. , Boddu , S. , Branco Lins , P. R., Brooks , L. , Ramaraju , S. B. , Campbell , L. I. , Martinez , M. C. , Charkhchi , M. , Cortes , L. A. , Davidson , C. , Denni , S. , Dodiya , K. , Donaldson , S. , El Houdaigui , B. , El Naboulsi , T. , Falola , O. , Fatima , R. , Genez , T. , Martinez , J. G. , Gurbich , T. , Hardy , M. , Hollis , Z. , Hunt , T. , Kay , M. , Kaykala , V. , Lemos , D. , Lodha , D. , Mathlouthi , N. , Merino , G. A. , Merritt , R. , Mirabueno , L. P. , Mushtaq , A. , Hossain , S. N. , Pérez-Silva , J. G. , Perry , M. , Piližota , I. , Poppleton , D. , Prosovetskaia , I. , Raj , S. , Salam , A. I. , Saraf , S. , Saraiva-Agostinho , N. , Sinha , S. , Sipos , B. , Sitnik , V. , Steed , E. , Suner , M.-M. , Surapaneni , L. , Sutinen , K. , Tricomi , F. F. , Tsang , I. , Urbina-Gómez , D. , Veidenberg , A. , Walsh , T. A. , Willhoft , N. L. , Allen , J. , Alvarez-Jarreta , J. , Chakiachvili , M. , Cheema , J. , da Rocha , J. B. , De Silva , N. H. , Giorgetti , S. , Haggerty , L. , Ilsley , G. R. , Keatley , J. , Loveland , J. E. , Moore , B. , Mudge , J. M. , Naamati , G. , Tate , J. , Trevanion , S. J. , Winterbottom , A. , Flint , B. , Frankish , A. , Hunt , S. E. , Finn , R. D. , Freeberg , M. A. , Harrison , P. W. , Martin , F. J. , and Yates , A. D. ( 2024 ). Ensembl 2025 . Nucleic Acids Research , 53 ( D1 ): D948 – D957 . OpenUrl CrossRef ↵ Harris , C. R. , Millman , K. J. , van der Walt , S. J. , Gommers , R. , Virtanen , P. , Cournapeau , D. , Wieser , E. , Taylor , J. , Berg , S. , Smith , N. J. , Kern , R. , Picus , M. , Hoyer , S. , van Kerkwijk , M. H. , Brett , M. , Haldane , A. , del Río , J. F. , Wiebe , M. , Peterson , P. , Gérard-Marchant , P. , Sheppard , K. , Reddy , T. , Weckesser , W. , Abbasi , H. , Gohlke , C. , and Oliphant , T. E. ( 2020 ). Array programming with NumPy . Nature , 585 ( 7825 ): 357 – 362 . OpenUrl CrossRef PubMed ↵ Kronopt ( 2020 ). fastaparser python module . https://github.com/Kronopt/FastaParser . ↵ Openvax ( 2015 ). gtfparse python module . https://github.com/openvax/gtfparse . ↵ Stovner , E. B. and Sætrom , P. ( 2019 ). PyRanges: efficient comparison of genomic intervals in Python . Bioinformatics , 36 ( 3 ): 918 – 919 . OpenUrl ↵ Warden , P. ( 2010 ). hashmap.h . https://github.com/sheredom/hashmap.h . View the discussion thread. Back to top Previous Next Posted April 27, 2025. Download PDF Email Thank you for your interest in spreading the word about bioRxiv. NOTE: Your email address is requested solely to identify you as the sender of this article. Your Email * Your Name * Send To * Enter multiple addresses on separate lines or separate them with commas. You are going to email the following Parsing GTF and FASTA files using the eccLib Library Message Subject (Your Name) has forwarded a page to you from bioRxiv Message Body (Your Name) thought you would like to see this page from the bioRxiv website. Your Personal Message CAPTCHA This question is for testing whether or not you are a human visitor and to prevent automated spam submissions. Share Parsing GTF and FASTA files using the eccLib Library Tomasz Chady , Zuzanna Karolina Filutowska bioRxiv 2025.04.24.650481; doi: https://doi.org/10.1101/2025.04.24.650481 Share This Article: Copy Citation Tools Parsing GTF and FASTA files using the eccLib Library Tomasz Chady , Zuzanna Karolina Filutowska bioRxiv 2025.04.24.650481; doi: https://doi.org/10.1101/2025.04.24.650481 Citation Manager Formats BibTeX Bookends EasyBib EndNote (tagged) EndNote 8 (xml) Medlars Mendeley Papers RefWorks Tagged Ref Manager RIS Zotero Tweet Widget Facebook Like Google Plus One Subject Area Bioinformatics Subject Areas All Articles Animal Behavior and Cognition (7624) Biochemistry (17650) Bioengineering (13871) Bioinformatics (41882) Biophysics (21424) Cancer Biology (18566) Cell Biology (25461) Clinical Trials (138) Developmental Biology (13365) Ecology (19867) Epidemiology (2067) Evolutionary Biology (24290) Genetics (15590) Genomics (22476) Immunology (17713) Microbiology (40331) Molecular Biology (17148) Neuroscience (88477) Paleontology (666) Pathology (2828) Pharmacology and Toxicology (4816) Physiology (7635) Plant Biology (15114) Scientific Communication and Education (2044) Synthetic Biology (4286) Systems Biology (9815) Zoology (2268)