transformer.py
def load_and_tokenize(self, file_path, tokenizer):
        """
        load a sequential text dataset and tokenize it line by line
        sample of file:
        event1, event2, event3
        event5, event3, event7
        ...
        """
        if not os.path.exists(file_path):
            print_red("File {0} not exists. Cannot find the input dataset file to tokenize.".format(file_path))
            return None

        tokenized_dataset = LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=10)
        return tokenized_dataset

        load a sequential text dataset and tokenize it line by line
        sample of file:
        event1, event2, event3
        event5, event3, event7
        ...
        
load a sequential text dataset and tokenize it line by line
sample of file:
event1, event2, event3
event5, event3, event7
load a sequential text dataset and tokenize it line by line sample of file : event1 event2 event3 event5 event3 event7
def load_and_tokenize(self, file_path, tokenizer):
        if not os.path.exists(file_path):
            print_red("File {0} not exists. Cannot find the input dataset file to tokenize.".format(file_path))
            return None
        tokenized_dataset = LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=10)
        return tokenized_dataset
def load_and_tokenize ( self , file_path , tokenizer ) : if not os . path . exists ( file_path ) : print_red ( "File {0} not exists. Cannot find the input dataset file to tokenize." . format ( file_path ) ) return None tokenized_dataset = LineByLineTextDataset ( tokenizer = tokenizer , file_path = file_path , block_size = 10 ) return tokenized_dataset
load a sequential text dataset and tokenize it line by line
sample of file:
event1, event2, event3
event5, event3, event7
...
load a sequential text dataset and tokenize it line by line sample of file : event1 event2 event3 event5 event3 event7 ...
"""
        load a sequential text dataset and tokenize it line by line
        sample of file:
        event1, event2, event3
        event5, event3, event7
        ...
        """
self
file_path
tokenizer