tokenizer.py
def train(self):
        """Learn the tokenization from the input files
        """
        #Get the vocabulary from the monolingual data
        vocabs = []
        if not self.use_vocab:
            pre_tokenizer = BertPreTokenizer()
            print("Getting the vocabulary from input files")
            print(self.use_vocab)
            for i,filename in enumerate(self.input_files):
                print(filename)
                with open(filename) as f:
                    vocab,sentences = get_vocabulary(f,pre_tokenizer,self.num_workers)
                    vocabs.append(vocab)
                    self.num_sentences[i] = sentences

        else:
            print("Getting vocabulary from vocab files")
            for i,filename in enumerate(self.vocab_files):
                print(filename)
                with open(filename,'rb') as f:
                    vocab,sentences = pickle.load(f)
                    vocabs.append(vocab)
                    self.num_sentences[i] = sentences

        #Process the vocab
        self.process_vocab(vocabs)

        #Get pairwise statistics
        self.get_pair_statistics()

        char_vocab = len(self.final_vocab)
        print("Character vocabulary size:",char_vocab)
        num_merges = self.vocab_size-char_vocab
        print("Number of merges required for %d vocab size: %d"%(self.vocab_size,num_merges))

        #Merge loop
        for m in tqdm(range(num_merges)):
            max_occurence = self.cost_heap.max()
            if max_occurence == -1:
                print("No more merges possible. Exiting loop")
                break

            assert (max_occurence in self.inv_cost_stats)
            pair = self.inv_cost_stats[max_occurence].pop()

            #Replace the existing pair with the merged pair
            changes,new_pair = self.replace_pair(pair)

            #Update the statistics table
            self.update_pair_statistics(pair,new_pair,changes)

            assert("".join(pair)==new_pair)
            self.final_vocab.append(new_pair)
            self.merges.append(pair)
Learn the tokenization from the input files
        
Learn the tokenization from the input files
Learn the tokenization from the input files
def train(self):
        vocabs = []
        if not self.use_vocab:
            pre_tokenizer = BertPreTokenizer()
            print("Getting the vocabulary from input files")
            print(self.use_vocab)
            for i,filename in enumerate(self.input_files):
                print(filename)
                with open(filename) as f:
                    vocab,sentences = get_vocabulary(f,pre_tokenizer,self.num_workers)
                    vocabs.append(vocab)
                    self.num_sentences[i] = sentences
        else:
            print("Getting vocabulary from vocab files")
            for i,filename in enumerate(self.vocab_files):
                print(filename)
                with open(filename,'rb') as f:
                    vocab,sentences = pickle.load(f)
                    vocabs.append(vocab)
                    self.num_sentences[i] = sentences
        self.process_vocab(vocabs)
        self.get_pair_statistics()
        char_vocab = len(self.final_vocab)
        print("Character vocabulary size:",char_vocab)
        num_merges = self.vocab_size-char_vocab
        print("Number of merges required for %d vocab size: %d"%(self.vocab_size,num_merges))
        for m in tqdm(range(num_merges)):
            max_occurence = self.cost_heap.max()
            if max_occurence == -1:
                print("No more merges possible. Exiting loop")
                break
            assert (max_occurence in self.inv_cost_stats)
            pair = self.inv_cost_stats[max_occurence].pop()
            changes,new_pair = self.replace_pair(pair)
            self.update_pair_statistics(pair,new_pair,changes)
            assert("".join(pair)==new_pair)
            self.final_vocab.append(new_pair)
            self.merges.append(pair)
def train ( self ) : vocabs = [ ] if not self . use_vocab : pre_tokenizer = BertPreTokenizer ( ) print ( "Getting the vocabulary from input files" ) print ( self . use_vocab ) for i , filename in enumerate ( self . input_files ) : print ( filename ) with open ( filename ) as f : vocab , sentences = get_vocabulary ( f , pre_tokenizer , self . num_workers ) vocabs . append ( vocab ) self . num_sentences [ i ] = sentences else : print ( "Getting vocabulary from vocab files" ) for i , filename in enumerate ( self . vocab_files ) : print ( filename ) with open ( filename , 'rb' ) as f : vocab , sentences = pickle . load ( f ) vocabs . append ( vocab ) self . num_sentences [ i ] = sentences self . process_vocab ( vocabs ) self . get_pair_statistics ( ) char_vocab = len ( self . final_vocab ) print ( "Character vocabulary size:" , char_vocab ) num_merges = self . vocab_size - char_vocab print ( "Number of merges required for %d vocab size: %d" % ( self . vocab_size , num_merges ) ) for m in tqdm ( range ( num_merges ) ) : max_occurence = self . cost_heap . max ( ) if max_occurence == - 1 : print ( "No more merges possible. Exiting loop" ) break assert ( max_occurence in self . inv_cost_stats ) pair = self . inv_cost_stats [ max_occurence ] . pop ( ) changes , new_pair = self . replace_pair ( pair ) self . update_pair_statistics ( pair , new_pair , changes ) assert ( "" . join ( pair ) == new_pair ) self . final_vocab . append ( new_pair ) self . merges . append ( pair )
Learn the tokenization from the input files
Learn the tokenization from the input files
"""Learn the tokenization from the input files
        """ #Get the vocabulary from the monolingual data #Process the vocab #Get pairwise statistics #Merge loop #Replace the existing pair with the merged pair #Update the statistics table
self