What is the fastest way to calculate the number of unknown nucleotides in FASTA / FASTQ files? I used to work with publicly available genomic references, where basic statistics are usually available and if they are not, you have to compute them only once so there is no reason to worry about performance.

Recently I started sequencing project of couple of different species with mid-sized genomes (~Gbp) and during testing of different assembly pipelines I had compute number of unknown nucleotides many times in both raw reads (in fastq) and assembly scaffolds (in fasta), therefore I thought that I would like to optimize the computation.


For me it is reasonable to expect 4-line formatted fastq files, but general solution is still prefered
It would be nice if solution would work on gzipped files as well


Q : What is the fastest way (performance-wise) to compute the number of unknown nucleotides (Ns) in fasta and fastq files?
For FASTQ:

seqtk fqchk in.fq | head -2


It gives you percentage of "N" bases, not the exact count, though.

For FASTA:

seqtk comp in.fa | awk '{x+=$9}END{print x}'


This command line also works with FASTQ, but it will be slower as awk is slow.

EDIT: ok, based on @BaCH's reminder, here we go (you need kseq.h to compile):

// to compile: gcc -O2 -o count-N this-prog.c -lz
#include &lt;zlib.h&gt;
#include &lt;stdio.h&gt;
#include &lt;stdint.h&gt;
#include "kseq.h"
KSEQ_INIT(gzFile, gzread)

unsigned char dna5tbl[256] = {
    0, 1, 2, 3,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 5, 4, 4,
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4
};

int main(int argc, char *argv[]) {
    long i, n_n = 0, n_acgt = 0, n_gap = 0;
    gzFile fp;
    kseq_t *seq;
    if (argc == 1) {
        fprintf(stderr, "Usage: count-N &lt;in.fa&gt;\n");
        return 1;
    }
    if ((fp = gzopen(argv[1], "r")) == 0) {
        fprintf(stderr, "ERROR: fail to open the input file\n");
        return 1;
    }
    seq = kseq_init(fp);
    while (kseq_read(seq) &gt;= 0) {
        for (i = 0; i &lt; seq-&gt;seq.l; ++i) {
            int c = dna5tbl[(unsigned char)seq-&gt;seq.s[i]];
            if (c &lt; 4) ++n_acgt;
            else if (c == 4) ++n_n;
            else ++n_gap;
        }
    }
    kseq_destroy(seq);
    gzclose(fp);
    printf("%ld\t%ld\t%ld\n", n_acgt, n_n, n_gap);
    return 0;
}


It works for both FASTA/Q and gzip'ed FASTA/Q. The following uses SeqAn:

#include &lt;seqan/seq_io.h&gt;

using namespace seqan;

int main(int argc, char *argv[]) {
    if (argc == 1) {
        std::cerr &lt;&lt; "Usage: count-N &lt;in.fastq&gt;" &lt;&lt; std::endl;
        return 1;
    }
    std::ios::sync_with_stdio(false);
    CharString id;
    Dna5String seq;
    SeqFileIn seqFileIn(argv[1]);
    long i, n_n = 0, n_acgt = 0;
    while (!atEnd(seqFileIn)) {
        readRecord(id, seq, seqFileIn);
        for (i = beginPosition(seq); i &lt; endPosition(seq); ++i)
            if (seq[i] &lt; 4) ++n_acgt;
            else ++n_n;
    }
    std::cout &lt;&lt; n_acgt &lt;&lt; '\t' &lt;&lt; n_n &lt;&lt; std::endl;
    return 0;
}


On a FASTQ with 4-million 150bp reads:


The C version: ~0.74 sec
The C++ version: ~2.15 sec
An older C version without a lookup table (see the previous edit): ~2.65 sec