tokenizer.py
def update_stats(self,pair,index,value):
        """Update the stats along with the other variables for the given pair
        Args:
            pair (tupe): Pair for which update needs to be made
            indices (int): Index for which the changes need to be made
            values (int): New value for the given indices
        """
        if pair not in self.stats:
            assert (pair not in self.cost_stats)

        self.stats[pair][index] += value

        new_cost = self.cost_function(pair)

        if pair in self.cost_stats:
            prev_cost = self.cost_stats[pair]
        else:
            prev_cost = None

        self.cost_stats[pair] = new_cost
        if prev_cost is not None:
            assert (prev_cost in self.inv_cost_stats)

            if prev_cost == new_cost:
                return

            self.inv_cost_stats[prev_cost].remove(pair)
            if not self.inv_cost_stats[prev_cost]:
                self.cost_heap.delete(prev_cost)

        if not self.inv_cost_stats[new_cost]:
            self.cost_heap.push(new_cost)

        self.inv_cost_stats[new_cost].add(pair)
Update the stats along with the other variables for the given pair
        Args:
            pair (tupe): Pair for which update needs to be made
            indices (int): Index for which the changes need to be made
            values (int): New value for the given indices
        
Update the stats along with the other variables for the given pair
Update the stats along with the other variables for the given pair
def update_stats(self,pair,index,value):
        if pair not in self.stats:
            assert (pair not in self.cost_stats)
        self.stats[pair][index] += value
        new_cost = self.cost_function(pair)
        if pair in self.cost_stats:
            prev_cost = self.cost_stats[pair]
        else:
            prev_cost = None
        self.cost_stats[pair] = new_cost
        if prev_cost is not None:
            assert (prev_cost in self.inv_cost_stats)
            if prev_cost == new_cost:
                return
            self.inv_cost_stats[prev_cost].remove(pair)
            if not self.inv_cost_stats[prev_cost]:
                self.cost_heap.delete(prev_cost)
        if not self.inv_cost_stats[new_cost]:
            self.cost_heap.push(new_cost)
        self.inv_cost_stats[new_cost].add(pair)
def update_stats ( self , pair , index , value ) : if pair not in self . stats : assert ( pair not in self . cost_stats ) self . stats [ pair ] [ index ] += value new_cost = self . cost_function ( pair ) if pair in self . cost_stats : prev_cost = self . cost_stats [ pair ] else : prev_cost = None self . cost_stats [ pair ] = new_cost if prev_cost is not None : assert ( prev_cost in self . inv_cost_stats ) if prev_cost == new_cost : return self . inv_cost_stats [ prev_cost ] . remove ( pair ) if not self . inv_cost_stats [ prev_cost ] : self . cost_heap . delete ( prev_cost ) if not self . inv_cost_stats [ new_cost ] : self . cost_heap . push ( new_cost ) self . inv_cost_stats [ new_cost ] . add ( pair )
Update the stats along with the other variables for the given pair
Update the stats along with the other variables for the given pair
"""Update the stats along with the other variables for the given pair
        Args:
            pair (tupe): Pair for which update needs to be made
            indices (int): Index for which the changes need to be made
            values (int): New value for the given indices
        """
self
pair
Pair for which update needs to be made
Pair for which update needs to be made
index
value
indices
Index for which the changes need to be made
Index for which the changes need to be made
values
New value for the given indices
New value for the given indices