simpletransformers/language_modeling/language_modeling_model.py
def load_and_cache_examples(self, file_path, evaluate=False, no_cache=False, verbose=True, silent=False):
        """
        Reads a text file from file_path and creates training features.

        Utility function for train() and eval() methods. Not intended to be used directly.
        """

        tokenizer = self.tokenizer
        args = self.args

        if not no_cache:
            no_cache = args.no_cache

        if not no_cache:
            os.makedirs(self.args.cache_dir, exist_ok=True)

        mode = "dev" if evaluate else "train"

        if args.dataset_class:
            CustomDataset = args.dataset_class
            return CustomDataset(tokenizer, args, file_path, mode, args.block_size)
        else:
            dataset_type = args.dataset_type
            if dataset_type == "text":
                return TextDataset(tokenizer, file_path, args.block_size, overwrite_cache=True)
            elif dataset_type == "line_by_line":
                return LineByLineTextDataset(tokenizer, file_path, args.block_size)
            else:
                special_tokens_count = 3 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 2
                if self.args.max_seq_length > 509 and self.args.model_type != "longformer":
                    self.args.max_seq_length = (
                        509 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 510
                    )
                    self.args.block_size = (
                        509 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 510
                    )
                return SimpleDataset(
                    tokenizer,
                    self.args,
                    file_path,
                    mode,
                    args.block_size,
                    special_tokens_count,
                    sliding_window=args.sliding_window,
                )

        Reads a text file from file_path and creates training features.

        Utility function for train() and eval() methods. Not intended to be used directly.
        
Reads a text file from file_path and creates training features.
Utility function for train() and eval() methods. Not intended to be used directly.
Reads a text file from file_path and creates training features . Utility function for train () and eval () methods . Not intended to be used directly .
def load_and_cache_examples(self, file_path, evaluate=False, no_cache=False, verbose=True, silent=False):
        tokenizer = self.tokenizer
        args = self.args
        if not no_cache:
            no_cache = args.no_cache
        if not no_cache:
            os.makedirs(self.args.cache_dir, exist_ok=True)
        mode = "dev" if evaluate else "train"
        if args.dataset_class:
            CustomDataset = args.dataset_class
            return CustomDataset(tokenizer, args, file_path, mode, args.block_size)
        else:
            dataset_type = args.dataset_type
            if dataset_type == "text":
                return TextDataset(tokenizer, file_path, args.block_size, overwrite_cache=True)
            elif dataset_type == "line_by_line":
                return LineByLineTextDataset(tokenizer, file_path, args.block_size)
            else:
                special_tokens_count = 3 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 2
                if self.args.max_seq_length > 509 and self.args.model_type != "longformer":
                    self.args.max_seq_length = (
                        509 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 510
                    )
                    self.args.block_size = (
                        509 if bool(args.model_type in ["roberta", "camembert", "xlmroberta"]) else 510
                    )
                return SimpleDataset(
                    tokenizer,
                    self.args,
                    file_path,
                    mode,
                    args.block_size,
                    special_tokens_count,
                    sliding_window=args.sliding_window,
                )
def load_and_cache_examples ( self , file_path , evaluate = False , no_cache = False , verbose = True , silent = False ) : tokenizer = self . tokenizer args = self . args if not no_cache : no_cache = args . no_cache if not no_cache : os . makedirs ( self . args . cache_dir , exist_ok = True ) mode = "dev" if evaluate else "train" if args . dataset_class : CustomDataset = args . dataset_class return CustomDataset ( tokenizer , args , file_path , mode , args . block_size ) else : dataset_type = args . dataset_type if dataset_type == "text" : return TextDataset ( tokenizer , file_path , args . block_size , overwrite_cache = True ) elif dataset_type == "line_by_line" : return LineByLineTextDataset ( tokenizer , file_path , args . block_size ) else : special_tokens_count = 3 if bool ( args . model_type in [ "roberta" , "camembert" , "xlmroberta" ] ) else 2 if self . args . max_seq_length > 509 and self . args . model_type != "longformer" : self . args . max_seq_length = ( 509 if bool ( args . model_type in [ "roberta" , "camembert" , "xlmroberta" ] ) else 510 ) self . args . block_size = ( 509 if bool ( args . model_type in [ "roberta" , "camembert" , "xlmroberta" ] ) else 510 ) return SimpleDataset ( tokenizer , self . args , file_path , mode , args . block_size , special_tokens_count , sliding_window = args . sliding_window , )
Reads a text file from file_path and creates training features.
Reads a text file from file_path and creates training features .
"""
        Reads a text file from file_path and creates training features.

        Utility function for train() and eval() methods. Not intended to be used directly.
        """
self
file_path
evaluate
no_cache
verbose
silent