tokenizer.py
def update_pair_statistics(self, pair, new_pair, changes):
        """Minimally update the indices and statistics; if we merge a pair of symbols,
       only pairs that overlap with occurrences of this pair are affected, and need to be updated.
        Args:
            pair (tuple): Pair to be merged
            new_pair (str): Merged pair to be used in place of the original pair
            changes (list of tuple): Changes made in the vocab file
        """
        self.delete_pair(pair)
        for i in range(self.n_lang):
            self.indices[i][pair] = defaultdict(int)
        first, second = pair
        for ind, j, word, old_word, freq in changes:

            # find all instances of pair, and update frequency/indices around it
            i = 0
            while True:
                # find first symbol
                try:
                    i = old_word.index(first, i)
                except ValueError:
                   break
                # if first symbol is followed by second symbol, we've found an occurrence of pair (old_word[i:i+2])
                if i < len(old_word)-1 and old_word[i+1] == second:
                    # assuming a symbol sequence "A B C", if "B C" is merged, reduce the frequency of "A B"
                    if i:
                        prev = tuple(old_word[i-1:i+1])
                        self.update_stats(prev,ind,-freq)
                        self.indices[ind][prev][j] -= 1
                    if i < len(old_word)-2:
                        # assuming a symbol sequence "A B C B", if "B C" is merged, reduce the frequency of "C B".
                        # however, skip this if the sequence is A B C B C, because the frequency of "C B" will be reduced by the previous code block
                        if old_word[i+2] != first or i >= len(old_word)-3 or old_word[i+3] != second:
                            nex = tuple(old_word[i+1:i+3])
                            self.update_stats(nex,ind,-freq)
                            self.indices[ind][nex][j] -= 1
                    i += 2
                else:
                    i += 1

            i = 0
            while True:
                try:
                    # find new pair
                    i = word.index(new_pair, i)
                except ValueError:
                    break
                # assuming a symbol sequence "A BC D", if "B C" is merged, increase the frequency of "A BC"
                if i:
                    prev = tuple(word[i-1:i+1])
                    self.update_stats(prev,ind,freq)
                    self.indices[ind][prev][j] += 1
                # assuming a symbol sequence "A BC B", if "B C" is merged, increase the frequency of "BC B"
                # however, if the sequence is A BC BC, skip this step because the count of "BC BC" will be incremented by the previous code block
                if i < len(word)-1 and word[i+1] != new_pair:
                    nex = tuple(word[i:i+2])
                    self.update_stats(nex,ind,freq)
                    self.indices[ind][nex][j] += 1
                i += 1
Minimally update the indices and statistics; if we merge a pair of symbols,
       only pairs that overlap with occurrences of this pair are affected, and need to be updated.
        Args:
            pair (tuple): Pair to be merged
            new_pair (str): Merged pair to be used in place of the original pair
            changes (list of tuple): Changes made in the vocab file
        
Minimally update the indices and statistics; if we merge a pair of symbols,
only pairs that overlap with occurrences of this pair are affected, and need to be updated.
Args:
pair (tuple): Pair to be merged
new_pair (str): Merged pair to be used in place of the original pair
changes (list of tuple): Changes made in the vocab file
Minimally update the indices and statistics ; if we merge a pair of symbols only pairs that overlap with occurrences of this pair are affected and need to be updated . Args : pair ( tuple ) : Pair to be merged new_pair ( str ) : Merged pair to be used in place of the original pair changes ( list of tuple ) : Changes made in the vocab file
def update_pair_statistics(self, pair, new_pair, changes):
        self.delete_pair(pair)
        for i in range(self.n_lang):
            self.indices[i][pair] = defaultdict(int)
        first, second = pair
        for ind, j, word, old_word, freq in changes:
            i = 0
            while True:
                try:
                    i = old_word.index(first, i)
                except ValueError:
                   break
                if i < len(old_word)-1 and old_word[i+1] == second:
                    if i:
                        prev = tuple(old_word[i-1:i+1])
                        self.update_stats(prev,ind,-freq)
                        self.indices[ind][prev][j] -= 1
                    if i < len(old_word)-2:
                        if old_word[i+2] != first or i >= len(old_word)-3 or old_word[i+3] != second:
                            nex = tuple(old_word[i+1:i+3])
                            self.update_stats(nex,ind,-freq)
                            self.indices[ind][nex][j] -= 1
                    i += 2
                else:
                    i += 1
            i = 0
            while True:
                try:
                    i = word.index(new_pair, i)
                except ValueError:
                    break
                if i:
                    prev = tuple(word[i-1:i+1])
                    self.update_stats(prev,ind,freq)
                    self.indices[ind][prev][j] += 1
                if i < len(word)-1 and word[i+1] != new_pair:
                    nex = tuple(word[i:i+2])
                    self.update_stats(nex,ind,freq)
                    self.indices[ind][nex][j] += 1
                i += 1
def update_pair_statistics ( self , pair , new_pair , changes ) : self . delete_pair ( pair ) for i in range ( self . n_lang ) : self . indices [ i ] [ pair ] = defaultdict ( int ) first , second = pair for ind , j , word , old_word , freq in changes : i = 0 while True : try : i = old_word . index ( first , i ) except ValueError : break if i < len ( old_word ) - 1 and old_word [ i + 1 ] == second : if i : prev = tuple ( old_word [ i - 1 : i + 1 ] ) self . update_stats ( prev , ind , - freq ) self . indices [ ind ] [ prev ] [ j ] -= 1 if i < len ( old_word ) - 2 : if old_word [ i + 2 ] != first or i >= len ( old_word ) - 3 or old_word [ i + 3 ] != second : nex = tuple ( old_word [ i + 1 : i + 3 ] ) self . update_stats ( nex , ind , - freq ) self . indices [ ind ] [ nex ] [ j ] -= 1 i += 2 else : i += 1 i = 0 while True : try : i = word . index ( new_pair , i ) except ValueError : break if i : prev = tuple ( word [ i - 1 : i + 1 ] ) self . update_stats ( prev , ind , freq ) self . indices [ ind ] [ prev ] [ j ] += 1 if i < len ( word ) - 1 and word [ i + 1 ] != new_pair : nex = tuple ( word [ i : i + 2 ] ) self . update_stats ( nex , ind , freq ) self . indices [ ind ] [ nex ] [ j ] += 1 i += 1
Minimally update the indices and statistics; if we merge a pair of symbols,
only pairs that overlap with occurrences of this pair are affected, and need to be updated.
Minimally update the indices and statistics ; if we merge a pair of symbols only pairs that overlap with occurrences of this pair are affected and need to be updated .
"""Minimally update the indices and statistics; if we merge a pair of symbols,
       only pairs that overlap with occurrences of this pair are affected, and need to be updated.
        Args:
            pair (tuple): Pair to be merged
            new_pair (str): Merged pair to be used in place of the original pair
            changes (list of tuple): Changes made in the vocab file
        """ # find all instances of pair, and update frequency/indices around it # find first symbol # if first symbol is followed by second symbol, we've found an occurrence of pair (old_word[i:i+2]) # assuming a symbol sequence "A B C", if "B C" is merged, reduce the frequency of "A B" # assuming a symbol sequence "A B C B", if "B C" is merged, reduce the frequency of "C B". # however, skip this if the sequence is A B C B C, because the frequency of "C B" will be reduced by the previous code block # find new pair # assuming a symbol sequence "A BC D", if "B C" is merged, increase the frequency of "A BC" # assuming a symbol sequence "A BC B", if "B C" is merged, increase the frequency of "BC B" # however, if the sequence is A BC BC, skip this step because the count of "BC BC" will be incremented by the previous code block
self
pair
new_pair
changes