nlpmodels/utils/elt/transformer_dataset.py
def build_token_lists(filepaths: List, de_tokenizer, en_tokenizer):
        """
        This is a staticmethod used to return the data from files in a tokenized format.

        Args:
            filepaths(List): A list of both source and target lists.
            de_tokenizer(function): The tokenizer function for German.
            en_tokenizer(function): The tokenizer function for English.

        Returns:
            A tuple containing the list of tokens for each example.
        """
        raw_de_iter = iter(io.open(filepaths[0], encoding="utf8"))
        raw_en_iter = iter(io.open(filepaths[1], encoding="utf8"))
        de_tokens_list = []
        en_tokens_list = []
        for (raw_de, raw_en) in zip(raw_de_iter, raw_en_iter):
            de_tokens = de_tokenizer([raw_de])[0]
            en_tokens = en_tokenizer([raw_en])[0]
            de_tokens_list.append(de_tokens)
            en_tokens_list.append(en_tokens)
        return de_tokens_list, en_tokens_list

        This is a staticmethod used to return the data from files in a tokenized format.

        Args:
            filepaths(List): A list of both source and target lists.
            de_tokenizer(function): The tokenizer function for German.
            en_tokenizer(function): The tokenizer function for English.

        Returns:
            A tuple containing the list of tokens for each example.
        
This is a staticmethod used to return the data from files in a tokenized format.
This is a staticmethod used to return the data from files in a tokenized format .
def build_token_lists(filepaths: List, de_tokenizer, en_tokenizer):
        raw_de_iter = iter(io.open(filepaths[0], encoding="utf8"))
        raw_en_iter = iter(io.open(filepaths[1], encoding="utf8"))
        de_tokens_list = []
        en_tokens_list = []
        for (raw_de, raw_en) in zip(raw_de_iter, raw_en_iter):
            de_tokens = de_tokenizer([raw_de])[0]
            en_tokens = en_tokenizer([raw_en])[0]
            de_tokens_list.append(de_tokens)
            en_tokens_list.append(en_tokens)
        return de_tokens_list, en_tokens_list
def build_token_lists ( filepaths : List , de_tokenizer , en_tokenizer ) : raw_de_iter = iter ( io . open ( filepaths [ 0 ] , encoding = "utf8" ) ) raw_en_iter = iter ( io . open ( filepaths [ 1 ] , encoding = "utf8" ) ) de_tokens_list = [ ] en_tokens_list = [ ] for ( raw_de , raw_en ) in zip ( raw_de_iter , raw_en_iter ) : de_tokens = de_tokenizer ( [ raw_de ] ) [ 0 ] en_tokens = en_tokenizer ( [ raw_en ] ) [ 0 ] de_tokens_list . append ( de_tokens ) en_tokens_list . append ( en_tokens ) return de_tokens_list , en_tokens_list
This is a staticmethod used to return the data from files in a tokenized format.
This is a staticmethod used to return the data from files in a tokenized format .
"""
        This is a staticmethod used to return the data from files in a tokenized format.

        Args:
            filepaths(List): A list of both source and target lists.
            de_tokenizer(function): The tokenizer function for German.
            en_tokenizer(function): The tokenizer function for English.

        Returns:
            A tuple containing the list of tokens for each example.
        """
A tuple containing the list of tokens for each example.
A tuple containing the list of tokens for each example .
filepaths
List
A list of both source and target lists.
A list of both source and target lists .
de_tokenizer
The tokenizer function for German.
The tokenizer function for German .
en_tokenizer
The tokenizer function for English.
The tokenizer function for English .