'Builds dictionary from the list of provided tokens.'

def add_to_dict(self, tokens):

    for token in tokens:
        self.freq[token] += 1
        if (token not in self.tok2ind):
            index = len(self.tok2ind)
            self.tok2ind[token] = index
            self.ind2tok[index] = token
