nlpmodels/utils/elt/transformer_dataset.py
def build_vocab(filepath: AnyStr, tokenizer):
        """
        This is a static method that builds an NLPVocabulary object from a file provided.

        Args:
            filepath(Anystr): This is a string for the filepath to open to build the vocab.
            tokenizer(function): This is a function to convert a list of strings into tokens.

        Returns:
            A NLPVocabulary object built off this list.
        """
        vocab = NLPVocabulary()
        with io.open(filepath, encoding="utf8") as f:
            for string_ in f:
                vocab.add_many(tokenizer([string_])[0])
        return vocab

        This is a static method that builds an NLPVocabulary object from a file provided.

        Args:
            filepath(Anystr): This is a string for the filepath to open to build the vocab.
            tokenizer(function): This is a function to convert a list of strings into tokens.

        Returns:
            A NLPVocabulary object built off this list.
        
This is a static method that builds an NLPVocabulary object from a file provided.
This is a static method that builds an NLPVocabulary object from a file provided .
def build_vocab(filepath: AnyStr, tokenizer):
        vocab = NLPVocabulary()
        with io.open(filepath, encoding="utf8") as f:
            for string_ in f:
                vocab.add_many(tokenizer([string_])[0])
        return vocab
def build_vocab ( filepath : AnyStr , tokenizer ) : vocab = NLPVocabulary ( ) with io . open ( filepath , encoding = "utf8" ) as f : for string_ in f : vocab . add_many ( tokenizer ( [ string_ ] ) [ 0 ] ) return vocab
This is a static method that builds an NLPVocabulary object from a file provided.
This is a static method that builds an NLPVocabulary object from a file provided .
"""
        This is a static method that builds an NLPVocabulary object from a file provided.

        Args:
            filepath(Anystr): This is a string for the filepath to open to build the vocab.
            tokenizer(function): This is a function to convert a list of strings into tokens.

        Returns:
            A NLPVocabulary object built off this list.
        """
A NLPVocabulary object built off this list.
A NLPVocabulary object built off this list .
filepath
AnyStr
This is a string for the filepath to open to build the vocab.
This is a string for the filepath to open to build the vocab .
tokenizer
This is a function to convert a list of strings into tokens.
This is a function to convert a list of strings into tokens .