research/brotlidump.py
def metablock(self):
        """Process the data.
        Relevant variables of self:
        numberOfBlockTypes[kind]: number of block types
        currentBlockTypes[kind]: current block types (=0)
        literalContextModes: the context modes for the literal block types
        currentBlockCounts[kind]: counters for block types
        blockTypeCodes[kind]: code for block type
        blockCountCodes[kind]: code for block count
        cmaps[kind]: the context maps (not for I)
        prefixCodes[kind][#]: the prefix codes
        lastDistances: the last four distances
        lastChars: the last two chars
        output: the result
        """
        print('Meta block contents'.center(60, '='))
        self.currentBlockTypes = {L:0, I:0, D:0, pL:1, pI:1, pD:1}
        self.lastDistances = deque([17,16,11,4], maxlen=4)
        #the current context mode is for block type 0
        self.contextMode = ContextModeKeeper(self.literalContextModes[0])
        wordList = WordList()

        #setup distance callback function
        def distanceCallback(symbol, extra):
            "callback function for displaying decoded distance"
            index, offset = symbol.value(extra)
            if index:
                #recent distance
                distance = self.lastDistances[-index]+offset
                return 'Distance: {}last{:+d}={}'.format(index, offset, distance)
            #absolute value
            if offset<=maxDistance:
                return 'Absolute value: {} (pos {})'.format(offset, maxDistance-offset)
            #word list value
            action, word = divmod(offset-maxDistance, 1<<wordList.NDBITS[copyLen])
            return '{}-{} gives word {},{} action {}'.format(
                offset, maxDistance, copyLen, word, action)
        for dpc in self.prefixCodes[D]: dpc.callback = distanceCallback

        blockLen = 0
        #there we go
        while blockLen<self.MLEN:
            #get insert&copy command
            litLen, copyLen, dist0Flag = self.verboseRead(
                self.prefixCodes[I][
                    self.figureBlockType(I)])
            #literal data
            for i in range(litLen):
                bt = self.figureBlockType(L)
                cm = self.contextMode.getIndex()
                ct = self.cmaps[L][bt<<6|cm]
                char = self.verboseRead(
                    self.prefixCodes[L][ct],
                    context='{},{}='.format(bt,cm))
                self.contextMode.add(char)
                self.output.append(char)
            blockLen += litLen
            #check if we're done
            if blockLen>=self.MLEN: return
            #distance
            #distances are computed relative to output length, at most window size
            maxDistance = min(len(self.output), self.windowSize)
            if dist0Flag:
                distance = self.lastDistances[-1]
            else:
                bt = self.figureBlockType(D)
                cm = {2:0, 3:1, 4:2}.get(copyLen, 3)
                ct = self.cmaps[D][bt<<2|cm]
                index, offset = self.verboseRead(
                    self.prefixCodes[D][ct],
                    context='{},{}='.format(bt,cm))
                distance = self.lastDistances[-index]+offset if index else offset
                if index==1 and offset==0:
                    #to make sure distance is not put in last distance list
                    dist0Flag = True
            if distance<=maxDistance:
                #copy from output
                for i in range(
                        maxDistance-distance,
                        maxDistance-distance+copyLen):
                    self.output.append(self.output[i])
                if not dist0Flag: self.lastDistances.append(distance)
                comment = 'Seen before'
            else:
                #fetch from wordlist
                newWord = wordList.word(copyLen, distance-maxDistance-1)
                self.output.extend(newWord)
                #adjust copyLen to reflect actual new data
                copyLen = len(newWord)
                comment = 'From wordlist'
            blockLen += copyLen
            print(' '*40,
                comment,
                ': "',
                outputFormatter(self.output[-copyLen:]),
                '"',
                sep='')
            self.contextMode.add(self.output[-2])
            self.contextMode.add(self.output[-1])
def metablock(self):
        """Process the data.
        Relevant variables of self:
        numberOfBlockTypes[kind]: number of block types
        currentBlockTypes[kind]: current block types (=0)
        literalContextModes: the context modes for the literal block types
        currentBlockCounts[kind]: counters for block types
        blockTypeCodes[kind]: code for block type
        blockCountCodes[kind]: code for block count
        cmaps[kind]: the context maps (not for I)
        prefixCodes[kind][#]: the prefix codes
        lastDistances: the last four distances
        lastChars: the last two chars
        output: the result
        """
        print('Meta block contents'.center(60, '='))
        self.currentBlockTypes = {L:0, I:0, D:0, pL:1, pI:1, pD:1}
        self.lastDistances = deque([17,16,11,4], maxlen=4)
        #the current context mode is for block type 0
        self.contextMode = ContextModeKeeper(self.literalContextModes[0])
        wordList = WordList()

        #setup distance callback function
        def distanceCallback(symbol, extra):
            "callback function for displaying decoded distance"
            index, offset = symbol.value(extra)
            if index:
                #recent distance
                distance = self.lastDistances[-index]+offset
                return 'Distance: {}last{:+d}={}'.format(index, offset, distance)
            #absolute value
            if offset<=maxDistance:
                return 'Absolute value: {} (pos {})'.format(offset, maxDistance-offset)
            #word list value
            action, word = divmod(offset-maxDistance, 1<<wordList.NDBITS[copyLen])
            return '{}-{} gives word {},{} action {}'.format(
                offset, maxDistance, copyLen, word, action)
        for dpc in self.prefixCodes[D]: dpc.callback = distanceCallback

        blockLen = 0
        #there we go
        while blockLen<self.MLEN:
            #get insert&copy command
            litLen, copyLen, dist0Flag = self.verboseRead(
                self.prefixCodes[I][
                    self.figureBlockType(I)])
            #literal data
            for i in range(litLen):
                bt = self.figureBlockType(L)
                cm = self.contextMode.getIndex()
                ct = self.cmaps[L][bt<<6|cm]
                char = self.verboseRead(
                    self.prefixCodes[L][ct],
                    context='{},{}='.format(bt,cm))
                self.contextMode.add(char)
                self.output.append(char)
            blockLen += litLen
            #check if we're done
            if blockLen>=self.MLEN: return
            #distance
            #distances are computed relative to output length, at most window size
            maxDistance = min(len(self.output), self.windowSize)
            if dist0Flag:
                distance = self.lastDistances[-1]
            else:
                bt = self.figureBlockType(D)
                cm = {2:0, 3:1, 4:2}.get(copyLen, 3)
                ct = self.cmaps[D][bt<<2|cm]
                index, offset = self.verboseRead(
                    self.prefixCodes[D][ct],
                    context='{},{}='.format(bt,cm))
                distance = self.lastDistances[-index]+offset if index else offset
                if index==1 and offset==0:
                    #to make sure distance is not put in last distance list
                    dist0Flag = True
            if distance<=maxDistance:
                #copy from output
                for i in range(
                        maxDistance-distance,
                        maxDistance-distance+copyLen):
                    self.output.append(self.output[i])
                if not dist0Flag: self.lastDistances.append(distance)
                comment = 'Seen before'
            else:
                #fetch from wordlist
                newWord = wordList.word(copyLen, distance-maxDistance-1)
                self.output.extend(newWord)
                #adjust copyLen to reflect actual new data
                copyLen = len(newWord)
                comment = 'From wordlist'
            blockLen += copyLen
            print(' '*40,
                comment,
                ': "',
                outputFormatter(self.output[-copyLen:]),
                '"',
                sep='')
            self.contextMode.add(self.output[-2])
            self.contextMode.add(self.output[-1])
def metablock ( self ) : print ( 'Meta block contents' . center ( 60 , '=' ) ) self . currentBlockTypes = { L : 0 , I : 0 , D : 0 , pL : 1 , pI : 1 , pD : 1 } self . lastDistances = deque ( [ 17 , 16 , 11 , 4 ] , maxlen = 4 ) #the current context mode is for block type 0 self . contextMode = ContextModeKeeper ( self . literalContextModes [ 0 ] ) wordList = WordList ( ) #setup distance callback function def distanceCallback ( symbol , extra ) : "callback function for displaying decoded distance" index , offset = symbol . value ( extra ) if index : #recent distance distance = self . lastDistances [ - index ] + offset return 'Distance: {}last{:+d}={}' . format ( index , offset , distance ) #absolute value if offset <= maxDistance : return 'Absolute value: {} (pos {})' . format ( offset , maxDistance - offset ) #word list value action , word = divmod ( offset - maxDistance , 1 << wordList . NDBITS [ copyLen ] ) return '{}-{} gives word {},{} action {}' . format ( offset , maxDistance , copyLen , word , action ) for dpc in self . prefixCodes [ D ] : dpc . callback = distanceCallback blockLen = 0 #there we go while blockLen < self . MLEN : #get insert&copy command litLen , copyLen , dist0Flag = self . verboseRead ( self . prefixCodes [ I ] [ self . figureBlockType ( I ) ] ) #literal data for i in range ( litLen ) : bt = self . figureBlockType ( L ) cm = self . contextMode . getIndex ( ) ct = self . cmaps [ L ] [ bt << 6 | cm ] char = self . verboseRead ( self . prefixCodes [ L ] [ ct ] , context = '{},{}=' . format ( bt , cm ) ) self . contextMode . add ( char ) self . output . append ( char ) blockLen += litLen #check if we're done if blockLen >= self . MLEN : return #distance #distances are computed relative to output length, at most window size maxDistance = min ( len ( self . output ) , self . windowSize ) if dist0Flag : distance = self . lastDistances [ - 1 ] else : bt = self . figureBlockType ( D ) cm = { 2 : 0 , 3 : 1 , 4 : 2 } . get ( copyLen , 3 ) ct = self . cmaps [ D ] [ bt << 2 | cm ] index , offset = self . verboseRead ( self . prefixCodes [ D ] [ ct ] , context = '{},{}=' . format ( bt , cm ) ) distance = self . lastDistances [ - index ] + offset if index else offset if index == 1 and offset == 0 : #to make sure distance is not put in last distance list dist0Flag = True if distance <= maxDistance : #copy from output for i in range ( maxDistance - distance , maxDistance - distance + copyLen ) : self . output . append ( self . output [ i ] ) if not dist0Flag : self . lastDistances . append ( distance ) comment = 'Seen before' else : #fetch from wordlist newWord = wordList . word ( copyLen , distance - maxDistance - 1 ) self . output . extend ( newWord ) #adjust copyLen to reflect actual new data copyLen = len ( newWord ) comment = 'From wordlist' blockLen += copyLen print ( ' ' * 40 , comment , ': "' , outputFormatter ( self . output [ - copyLen : ] ) , '"' , sep = '' ) self . contextMode . add ( self . output [ - 2 ] ) self . contextMode . add ( self . output [ - 1 ] )
Process the data.
        Relevant variables of self:
        numberOfBlockTypes[kind]: number of block types
        currentBlockTypes[kind]: current block types (=0)
        literalContextModes: the context modes for the literal block types
        currentBlockCounts[kind]: counters for block types
        blockTypeCodes[kind]: code for block type
        blockCountCodes[kind]: code for block count
        cmaps[kind]: the context maps (not for I)
        prefixCodes[kind][#]: the prefix codes
        lastDistances: the last four distances
        lastChars: the last two chars
        output: the result
Process the data . Relevant variables of self : numberOfBlockTypes [ kind ] : number of block types currentBlockTypes [ kind ] : current block types ( = 0 ) literalContextModes : the context modes for the literal block types currentBlockCounts [ kind ] : counters for block types blockTypeCodes [ kind ] : code for block type blockCountCodes [ kind ] : code for block count cmaps [ kind ] : the context maps ( not for I ) prefixCodes [ kind ] [ # ] : the prefix codes lastDistances : the last four distances lastChars : the last two chars output : the result