puretabix/tabix.py
def build_from(cls, rawfile):
        """
        read a vcf file in blockgzip format and create an index object for it
        """

        file_format = 2  # 0 generic, 1 sam, 2 vcf
        column_sequence = 1  # column for sequence ids, 1-based
        column_begin = 2  # column for region start, 1-based
        column_end = 0  # column for region end, 1-based
        meta = "#"

        # dictionary of names to (bin_index, interval_index)
        # bin_index is a dictionary of bin numbers to [(chunk_start, chunk_end),...]
        # interval_index is a list of 16kbase interval start locations
        indexes = {}

        # go through each line in turn
        # parse it into a structured object
        vcf_fsm = get_vcf_fsm()
        accumulator = VCFAccumulator()

        # these are the internal two index types
        bin_index = {}
        interval_index = {}

        bgzipped = BlockGZipReader(rawfile)
        bgzipped.seek(0)

        for (
            start_block,
            start_offset,
            end_block,
            end_offset,
            line,
        ) in bgzipped.generate_lines_offset():
            line = line.decode() + "\n"
            vcf_fsm.run(line, LINE_START, accumulator)
            vcf_line = accumulator.to_vcfline()
            accumulator.reset()
            # TODO add better debugging for unexpected lines

            # skip any comment lines with # or ##
            if vcf_line.comment_raw or vcf_line.comment_key:
                continue

            # have we started a new chromosome?
            if vcf_line.chrom not in indexes:
                indexes[vcf_line.chrom] = ({}, [])
                bin_index = indexes[vcf_line.chrom][0]
                interval_index = indexes[vcf_line.chrom][1]

            # get the combined number for the block & offset
            start_virtual = start_block << 16 | start_offset
            end_virtual = end_block << 16 | end_offset

            # subtract 1 because its 0 offset
            record_start = vcf_line.pos - 1
            # subtract another 1 because half-open end
            record_end = vcf_line.pos - 1 + len(vcf_line.ref) - 1

            # bin index
            # smallest bin that completely contains the record

            bin_i = cls.region_to_bin(record_start, record_end)

            if bin_i not in bin_index.keys():
                bin_index[bin_i] = [(start_virtual, end_virtual + 1)]
            else:
                # extend chunk if directly continuous
                if bin_index[bin_i][-1][1] == start_virtual:
                    bin_index[bin_i][-1] = (
                        bin_index[bin_i][-1][0],
                        end_virtual + 1,
                    )
                else:
                    bin_index[bin_i].append((start_virtual, end_virtual + 1))

            # interval index
            # is the lowest virtual offset of all records that overlap interval
            # half-closed half-open records

            # throw away the first 14 bits to get interval index
            # e.g. 0 => 0, 16384 => 1, etc
            interval_i_start = record_start >> 14
            # subtract another 1 because half-open end
            interval_i_end = record_end >> 14

            for linear_i in sorted(set((interval_i_start, interval_i_end))):
                # add any padding intervals
                while len(interval_index) <= linear_i:
                    interval_index.append(-1)

                # if there is no previous value, or the previous value was higher
                # replace previous value
                if (
                    interval_index[linear_i] == -1
                    or start_virtual < interval_index[linear_i]
                ):
                    interval_index[linear_i] = start_virtual

            # pass through again and replace any -1 values with the last "not -1" value
            # these are intervals with no overlapping records
            lastgood = -1
            for i in range(len(interval_index)):
                if interval_index[i] == -1:
                    interval_index[i] = lastgood
                else:
                    lastgood = interval_index[i]

        return cls(
            file_format, column_sequence, column_begin, column_end, meta, 0, indexes
        )

        read a vcf file in blockgzip format and create an index object for it
        
read a vcf file in blockgzip format and create an index object for it
read a vcf file in blockgzip format and create an index object for it
def build_from(cls, rawfile):
        file_format = 2  
        column_sequence = 1  
        column_begin = 2  
        column_end = 0  
        meta = "#"
        indexes = {}
        vcf_fsm = get_vcf_fsm()
        accumulator = VCFAccumulator()
        bin_index = {}
        interval_index = {}
        bgzipped = BlockGZipReader(rawfile)
        bgzipped.seek(0)
        for (
            start_block,
            start_offset,
            end_block,
            end_offset,
            line,
        ) in bgzipped.generate_lines_offset():
            line = line.decode() + "\n"
            vcf_fsm.run(line, LINE_START, accumulator)
            vcf_line = accumulator.to_vcfline()
            accumulator.reset()
            if vcf_line.comment_raw or vcf_line.comment_key:
                continue
            if vcf_line.chrom not in indexes:
                indexes[vcf_line.chrom] = ({}, [])
                bin_index = indexes[vcf_line.chrom][0]
                interval_index = indexes[vcf_line.chrom][1]
            start_virtual = start_block << 16 | start_offset
            end_virtual = end_block << 16 | end_offset
            record_start = vcf_line.pos - 1
            record_end = vcf_line.pos - 1 + len(vcf_line.ref) - 1
            bin_i = cls.region_to_bin(record_start, record_end)
            if bin_i not in bin_index.keys():
                bin_index[bin_i] = [(start_virtual, end_virtual + 1)]
            else:
                if bin_index[bin_i][-1][1] == start_virtual:
                    bin_index[bin_i][-1] = (
                        bin_index[bin_i][-1][0],
                        end_virtual + 1,
                    )
                else:
                    bin_index[bin_i].append((start_virtual, end_virtual + 1))
            interval_i_start = record_start >> 14
            interval_i_end = record_end >> 14
            for linear_i in sorted(set((interval_i_start, interval_i_end))):
                while len(interval_index) <= linear_i:
                    interval_index.append(-1)
                if (
                    interval_index[linear_i] == -1
                    or start_virtual < interval_index[linear_i]
                ):
                    interval_index[linear_i] = start_virtual
            lastgood = -1
            for i in range(len(interval_index)):
                if interval_index[i] == -1:
                    interval_index[i] = lastgood
                else:
                    lastgood = interval_index[i]
        return cls(
            file_format, column_sequence, column_begin, column_end, meta, 0, indexes
        )
def build_from ( cls , rawfile ) : file_format = 2 column_sequence = 1 column_begin = 2 column_end = 0 meta = "#" indexes = { } vcf_fsm = get_vcf_fsm ( ) accumulator = VCFAccumulator ( ) bin_index = { } interval_index = { } bgzipped = BlockGZipReader ( rawfile ) bgzipped . seek ( 0 ) for ( start_block , start_offset , end_block , end_offset , line , ) in bgzipped . generate_lines_offset ( ) : line = line . decode ( ) + "\n" vcf_fsm . run ( line , LINE_START , accumulator ) vcf_line = accumulator . to_vcfline ( ) accumulator . reset ( ) if vcf_line . comment_raw or vcf_line . comment_key : continue if vcf_line . chrom not in indexes : indexes [ vcf_line . chrom ] = ( { } , [ ] ) bin_index = indexes [ vcf_line . chrom ] [ 0 ] interval_index = indexes [ vcf_line . chrom ] [ 1 ] start_virtual = start_block << 16 | start_offset end_virtual = end_block << 16 | end_offset record_start = vcf_line . pos - 1 record_end = vcf_line . pos - 1 + len ( vcf_line . ref ) - 1 bin_i = cls . region_to_bin ( record_start , record_end ) if bin_i not in bin_index . keys ( ) : bin_index [ bin_i ] = [ ( start_virtual , end_virtual + 1 ) ] else : if bin_index [ bin_i ] [ - 1 ] [ 1 ] == start_virtual : bin_index [ bin_i ] [ - 1 ] = ( bin_index [ bin_i ] [ - 1 ] [ 0 ] , end_virtual + 1 , ) else : bin_index [ bin_i ] . append ( ( start_virtual , end_virtual + 1 ) ) interval_i_start = record_start >> 14 interval_i_end = record_end >> 14 for linear_i in sorted ( set ( ( interval_i_start , interval_i_end ) ) ) : while len ( interval_index ) <= linear_i : interval_index . append ( - 1 ) if ( interval_index [ linear_i ] == - 1 or start_virtual < interval_index [ linear_i ] ) : interval_index [ linear_i ] = start_virtual lastgood = - 1 for i in range ( len ( interval_index ) ) : if interval_index [ i ] == - 1 : interval_index [ i ] = lastgood else : lastgood = interval_index [ i ] return cls ( file_format , column_sequence , column_begin , column_end , meta , 0 , indexes )
read a vcf file in blockgzip format and create an index object for it
read a vcf file in blockgzip format and create an index object for it
"""
        read a vcf file in blockgzip format and create an index object for it
        """ # 0 generic, 1 sam, 2 vcf # column for sequence ids, 1-based # column for region start, 1-based # column for region end, 1-based # dictionary of names to (bin_index, interval_index) # bin_index is a dictionary of bin numbers to [(chunk_start, chunk_end),...] # interval_index is a list of 16kbase interval start locations # go through each line in turn # parse it into a structured object # these are the internal two index types # TODO add better debugging for unexpected lines # skip any comment lines with # or ## # have we started a new chromosome? # get the combined number for the block & offset # subtract 1 because its 0 offset # subtract another 1 because half-open end # bin index # smallest bin that completely contains the record # extend chunk if directly continuous # interval index # is the lowest virtual offset of all records that overlap interval # half-closed half-open records # throw away the first 14 bits to get interval index # e.g. 0 => 0, 16384 => 1, etc # subtract another 1 because half-open end # add any padding intervals # if there is no previous value, or the previous value was higher # replace previous value # pass through again and replace any -1 values with the last "not -1" value # these are intervals with no overlapping records
cls
rawfile