research/brotlidump.py
def readComplexCode(self, hskip, alphabet):
        """Read complex code"""
        stream = self.stream
        #read the lengths for the length code
        lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:]
        codeLengths = {}
        total = 0
        lol = LengthOfLengthAlphabet('##'+alphabet.name)
        #lengthCode will be used for coding the lengths of the new code
        #we use it for display until now; definition comes below
        lengthCode = LengthAlphabet('#'+alphabet.name)
        lengthIter = iter(lengths)
        lengthsLeft = len(lengths)
        while total<32 and lengthsLeft>0:
            lengthsLeft -= 1
            newSymbol = next(lengthIter)
            lol.description = str(lengthCode[newSymbol])
            length = self.verboseRead(lol)
            if length:
                codeLengths[newSymbol] = length
                total += 32>>length
        if total>32: raise ValueError("Stream format")
        if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0
        #Now set the encoding of the lengthCode
        lengthCode.setLength(codeLengths)
        print("***** Lengths for {} will be coded as:".format(alphabet.name))
        lengthCode.showCode()
        #Now determine the symbol lengths with the lengthCode
        symbolLengths = {}
        total = 0
        lastLength = 8
        alphabetIter = iter(alphabet)
        while total<32768:
            #look ahead to see what is going to happen
            length = lengthCode.decodePeek(
                self.stream.peek(lengthCode.maxLength))[1].index
            #in every branch, set lengthCode.description to explanatory text
            #lengthCode calls format(symbol, extra) with this string
            if length==0:
                symbol = next(alphabetIter)
                lengthCode.description = 'symbol {} unused'.format(symbol)
                self.verboseRead(lengthCode)
                #unused symbol
                continue
            if length==16:
                lengthCode.description = \
                    '{1}+3 symbols of length '+str(lastLength)
                extra = self.verboseRead(lengthCode)
                #scan series of 16s (repeat counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                symbolLengths[startSymbol.index] = \
                    symbolLengths[endSymbol.index] = lastLength
                #count the two just defined symbols
                total += 2*32768>>lastLength
                #note: loop may end because we're there
                #even if a 16 _appears_ to follow
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<2)+extra+3
                    #read as many symbols as repeat increased
                    for i in range(oldRepeat, repeat):
                        endSymbol = next(alphabetIter)
                        symbolLengths[endSymbol.index] = lastLength
                    #compute new total; it may be end of loop
                    total += (repeat-oldRepeat)*32768>>lastLength
                    if total>=32768: break
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=16: break
                    lengthCode.description = 'total {}+{{1}} symbols'.format(
                        (repeat-2<<2)+3)
                    extra = self.verboseRead(lengthCode)
            elif length==17:
                #read, and show explanation
                lengthCode.description = '{1}+3 unused'
                extra = self.verboseRead(lengthCode)
                #scan series of 17s (groups of zero counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                #note: loop will not end with total==32768,
                #since total doesn't change here
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<3)+extra+3
                    #read as many symbols as repeat increases
                    for i in range(repeat-oldRepeat):
                        endSymbol = next(alphabetIter)
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=17: break
                    lengthCode.description = 'total {}+{{1}} unused'.format(
                        (repeat-2<<3)+3)
                    extra = self.verboseRead(lengthCode)
            else:
                symbol = next(alphabetIter)
                #double braces for format
                char = str(symbol)
                if char in '{}': char *= 2
                lengthCode.description = \
                    'Length for {} is {{0.index}} bits'.format(char)
                #output is not needed (will be 0)
                self.verboseRead(lengthCode)
                symbolLengths[symbol.index] = length
                total += 32768>>length
                lastLength = length
        assert total==32768
        alphabet.setLength(symbolLengths)
        print('End of table. Prefix code '+alphabet.name+':')
        alphabet.showCode()
def readComplexCode(self, hskip, alphabet):
        """Read complex code"""
        stream = self.stream
        #read the lengths for the length code
        lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:]
        codeLengths = {}
        total = 0
        lol = LengthOfLengthAlphabet('##'+alphabet.name)
        #lengthCode will be used for coding the lengths of the new code
        #we use it for display until now; definition comes below
        lengthCode = LengthAlphabet('#'+alphabet.name)
        lengthIter = iter(lengths)
        lengthsLeft = len(lengths)
        while total<32 and lengthsLeft>0:
            lengthsLeft -= 1
            newSymbol = next(lengthIter)
            lol.description = str(lengthCode[newSymbol])
            length = self.verboseRead(lol)
            if length:
                codeLengths[newSymbol] = length
                total += 32>>length
        if total>32: raise ValueError("Stream format")
        if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0
        #Now set the encoding of the lengthCode
        lengthCode.setLength(codeLengths)
        print("***** Lengths for {} will be coded as:".format(alphabet.name))
        lengthCode.showCode()
        #Now determine the symbol lengths with the lengthCode
        symbolLengths = {}
        total = 0
        lastLength = 8
        alphabetIter = iter(alphabet)
        while total<32768:
            #look ahead to see what is going to happen
            length = lengthCode.decodePeek(
                self.stream.peek(lengthCode.maxLength))[1].index
            #in every branch, set lengthCode.description to explanatory text
            #lengthCode calls format(symbol, extra) with this string
            if length==0:
                symbol = next(alphabetIter)
                lengthCode.description = 'symbol {} unused'.format(symbol)
                self.verboseRead(lengthCode)
                #unused symbol
                continue
            if length==16:
                lengthCode.description = \
                    '{1}+3 symbols of length '+str(lastLength)
                extra = self.verboseRead(lengthCode)
                #scan series of 16s (repeat counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                symbolLengths[startSymbol.index] = \
                    symbolLengths[endSymbol.index] = lastLength
                #count the two just defined symbols
                total += 2*32768>>lastLength
                #note: loop may end because we're there
                #even if a 16 _appears_ to follow
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<2)+extra+3
                    #read as many symbols as repeat increased
                    for i in range(oldRepeat, repeat):
                        endSymbol = next(alphabetIter)
                        symbolLengths[endSymbol.index] = lastLength
                    #compute new total; it may be end of loop
                    total += (repeat-oldRepeat)*32768>>lastLength
                    if total>=32768: break
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=16: break
                    lengthCode.description = 'total {}+{{1}} symbols'.format(
                        (repeat-2<<2)+3)
                    extra = self.verboseRead(lengthCode)
            elif length==17:
                #read, and show explanation
                lengthCode.description = '{1}+3 unused'
                extra = self.verboseRead(lengthCode)
                #scan series of 17s (groups of zero counts)
                #start with repeat count 2
                repeat = 2
                startSymbol = next(alphabetIter)
                endSymbol = next(alphabetIter)
                #note: loop will not end with total==32768,
                #since total doesn't change here
                while True:
                    #determine last symbol
                    oldRepeat = repeat
                    repeat = (repeat-2<<3)+extra+3
                    #read as many symbols as repeat increases
                    for i in range(repeat-oldRepeat):
                        endSymbol = next(alphabetIter)
                    #see if there is more to do
                    length = lengthCode.decodePeek(
                        self.stream.peek(lengthCode.maxLength))[1].index
                    if length!=17: break
                    lengthCode.description = 'total {}+{{1}} unused'.format(
                        (repeat-2<<3)+3)
                    extra = self.verboseRead(lengthCode)
            else:
                symbol = next(alphabetIter)
                #double braces for format
                char = str(symbol)
                if char in '{}': char *= 2
                lengthCode.description = \
                    'Length for {} is {{0.index}} bits'.format(char)
                #output is not needed (will be 0)
                self.verboseRead(lengthCode)
                symbolLengths[symbol.index] = length
                total += 32768>>length
                lastLength = length
        assert total==32768
        alphabet.setLength(symbolLengths)
        print('End of table. Prefix code '+alphabet.name+':')
        alphabet.showCode()
def readComplexCode ( self , hskip , alphabet ) : stream = self . stream #read the lengths for the length code lengths = [ 1 , 2 , 3 , 4 , 0 , 5 , 17 , 6 , 16 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 ] [ hskip : ] codeLengths = { } total = 0 lol = LengthOfLengthAlphabet ( '##' + alphabet . name ) #lengthCode will be used for coding the lengths of the new code #we use it for display until now; definition comes below lengthCode = LengthAlphabet ( '#' + alphabet . name ) lengthIter = iter ( lengths ) lengthsLeft = len ( lengths ) while total < 32 and lengthsLeft > 0 : lengthsLeft -= 1 newSymbol = next ( lengthIter ) lol . description = str ( lengthCode [ newSymbol ] ) length = self . verboseRead ( lol ) if length : codeLengths [ newSymbol ] = length total += 32 >> length if total > 32 : raise ValueError ( "Stream format" ) if len ( codeLengths ) == 1 : codeLengths [ list ( codeLengths . keys ( ) ) [ 0 ] ] = 0 #Now set the encoding of the lengthCode lengthCode . setLength ( codeLengths ) print ( "***** Lengths for {} will be coded as:" . format ( alphabet . name ) ) lengthCode . showCode ( ) #Now determine the symbol lengths with the lengthCode symbolLengths = { } total = 0 lastLength = 8 alphabetIter = iter ( alphabet ) while total < 32768 : #look ahead to see what is going to happen length = lengthCode . decodePeek ( self . stream . peek ( lengthCode . maxLength ) ) [ 1 ] . index #in every branch, set lengthCode.description to explanatory text #lengthCode calls format(symbol, extra) with this string if length == 0 : symbol = next ( alphabetIter ) lengthCode . description = 'symbol {} unused' . format ( symbol ) self . verboseRead ( lengthCode ) #unused symbol continue if length == 16 : lengthCode . description = '{1}+3 symbols of length ' + str ( lastLength ) extra = self . verboseRead ( lengthCode ) #scan series of 16s (repeat counts) #start with repeat count 2 repeat = 2 startSymbol = next ( alphabetIter ) endSymbol = next ( alphabetIter ) symbolLengths [ startSymbol . index ] = symbolLengths [ endSymbol . index ] = lastLength #count the two just defined symbols total += 2 * 32768 >> lastLength #note: loop may end because we're there #even if a 16 _appears_ to follow while True : #determine last symbol oldRepeat = repeat repeat = ( repeat - 2 << 2 ) + extra + 3 #read as many symbols as repeat increased for i in range ( oldRepeat , repeat ) : endSymbol = next ( alphabetIter ) symbolLengths [ endSymbol . index ] = lastLength #compute new total; it may be end of loop total += ( repeat - oldRepeat ) * 32768 >> lastLength if total >= 32768 : break #see if there is more to do length = lengthCode . decodePeek ( self . stream . peek ( lengthCode . maxLength ) ) [ 1 ] . index if length != 16 : break lengthCode . description = 'total {}+{{1}} symbols' . format ( ( repeat - 2 << 2 ) + 3 ) extra = self . verboseRead ( lengthCode ) elif length == 17 : #read, and show explanation lengthCode . description = '{1}+3 unused' extra = self . verboseRead ( lengthCode ) #scan series of 17s (groups of zero counts) #start with repeat count 2 repeat = 2 startSymbol = next ( alphabetIter ) endSymbol = next ( alphabetIter ) #note: loop will not end with total==32768, #since total doesn't change here while True : #determine last symbol oldRepeat = repeat repeat = ( repeat - 2 << 3 ) + extra + 3 #read as many symbols as repeat increases for i in range ( repeat - oldRepeat ) : endSymbol = next ( alphabetIter ) #see if there is more to do length = lengthCode . decodePeek ( self . stream . peek ( lengthCode . maxLength ) ) [ 1 ] . index if length != 17 : break lengthCode . description = 'total {}+{{1}} unused' . format ( ( repeat - 2 << 3 ) + 3 ) extra = self . verboseRead ( lengthCode ) else : symbol = next ( alphabetIter ) #double braces for format char = str ( symbol ) if char in '{}' : char *= 2 lengthCode . description = 'Length for {} is {{0.index}} bits' . format ( char ) #output is not needed (will be 0) self . verboseRead ( lengthCode ) symbolLengths [ symbol . index ] = length total += 32768 >> length lastLength = length assert total == 32768 alphabet . setLength ( symbolLengths ) print ( 'End of table. Prefix code ' + alphabet . name + ':' ) alphabet . showCode ( )
Read complex code
Read complex code