tokenizer.py
def process_vocab(self,vocabs):
        """Process the vocabulary obtained from the monolingual data (Splitting the words and adding end of suffix etc.)
        Args:
            vocabs (List of Counters): List of vocab files obatined from the monolingual data
        """

        print("Processing the vocabulary")
        original_chars = set()
        processed_chars = set()
        for i,vocab in enumerate(vocabs):
            print("Input file:",i)
            vocab_dict = {}
            for word,freq in tqdm(vocab.items()):
                new_word = []
                for j,x in enumerate(word):
                    original_chars.add(x)
                    x_ = x
                    if j!=0:
                        x_ = self.cw_prefix+x_
                    if j==len(word)-1:
                        x_ = x_ + self.eow_suffix
                    if x_!=x:
                        processed_chars.add(x_)
                    new_word.append(x_)
                    self.tok_counts[x_][i]+=freq
                vocab_dict[tuple(new_word)] = freq
            self.ind_vocabs.append(sorted(vocab_dict.items(),key=lambda x: x[1], reverse=True))

        self.final_vocab.extend(list(original_chars))
        self.final_vocab.extend(list(processed_chars))
Process the vocabulary obtained from the monolingual data (Splitting the words and adding end of suffix etc.)
        Args:
            vocabs (List of Counters): List of vocab files obatined from the monolingual data
        
Process the vocabulary obtained from the monolingual data (Splitting the words and adding end of suffix etc.)
Process the vocabulary obtained from the monolingual data ( Splitting the words and adding end of suffix etc . )
def process_vocab(self,vocabs):
        print("Processing the vocabulary")
        original_chars = set()
        processed_chars = set()
        for i,vocab in enumerate(vocabs):
            print("Input file:",i)
            vocab_dict = {}
            for word,freq in tqdm(vocab.items()):
                new_word = []
                for j,x in enumerate(word):
                    original_chars.add(x)
                    x_ = x
                    if j!=0:
                        x_ = self.cw_prefix+x_
                    if j==len(word)-1:
                        x_ = x_ + self.eow_suffix
                    if x_!=x:
                        processed_chars.add(x_)
                    new_word.append(x_)
                    self.tok_counts[x_][i]+=freq
                vocab_dict[tuple(new_word)] = freq
            self.ind_vocabs.append(sorted(vocab_dict.items(),key=lambda x: x[1], reverse=True))
        self.final_vocab.extend(list(original_chars))
        self.final_vocab.extend(list(processed_chars))
def process_vocab ( self , vocabs ) : print ( "Processing the vocabulary" ) original_chars = set ( ) processed_chars = set ( ) for i , vocab in enumerate ( vocabs ) : print ( "Input file:" , i ) vocab_dict = { } for word , freq in tqdm ( vocab . items ( ) ) : new_word = [ ] for j , x in enumerate ( word ) : original_chars . add ( x ) x_ = x if j != 0 : x_ = self . cw_prefix + x_ if j == len ( word ) - 1 : x_ = x_ + self . eow_suffix if x_ != x : processed_chars . add ( x_ ) new_word . append ( x_ ) self . tok_counts [ x_ ] [ i ] += freq vocab_dict [ tuple ( new_word ) ] = freq self . ind_vocabs . append ( sorted ( vocab_dict . items ( ) , key = lambda x : x [ 1 ] , reverse = True ) ) self . final_vocab . extend ( list ( original_chars ) ) self . final_vocab . extend ( list ( processed_chars ) )
Process the vocabulary obtained from the monolingual data (Splitting the words and adding end of suffix etc.)
Process the vocabulary obtained from the monolingual data ( Splitting the words and adding end of suffix etc . )
"""Process the vocabulary obtained from the monolingual data (Splitting the words and adding end of suffix etc.)
        Args:
            vocabs (List of Counters): List of vocab files obatined from the monolingual data
        """
self
vocabs
List of vocab files obatined from the monolingual data
List of vocab files obatined from the monolingual data