simpletransformers/language_modeling/language_modeling_model.py
def evaluate(self, eval_dataset, output_dir, multi_label=False, prefix="", verbose=True, silent=False, **kwargs):
        """
        Evaluates the model on eval_dataset.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        """

        model = self.model
        args = self.args
        eval_output_dir = output_dir
        tokenizer = self.tokenizer

        results = {}

        def collate(examples: List[torch.Tensor]):
            if tokenizer._pad_token is None:
                return pad_sequence(examples, batch_first=True)
            return pad_sequence(examples, batch_first=True, padding_value=tokenizer.pad_token_id)

        eval_sampler = SequentialSampler(eval_dataset)
        eval_dataloader = DataLoader(
            eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate
        )

        if args.n_gpu > 1:
            model = torch.nn.DataParallel(model)

        eval_loss = 0.0
        nb_eval_steps = 0
        model.eval()

        for batch in tqdm(eval_dataloader, disable=args.silent or silent, desc="Running Evaluation"):
            inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch)
            inputs = inputs.to(self.device)
            labels = labels.to(self.device)
            with torch.no_grad():
                outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model(inputs, labels=labels)
                if args.model_type == "electra":
                    g_loss = outputs[0]
                    d_loss = outputs[1]
                    lm_loss = g_loss + args.discriminator_loss_weight * d_loss
                else:
                    lm_loss = outputs[0]
                eval_loss += lm_loss.mean().item()
            nb_eval_steps += 1

        eval_loss = eval_loss / nb_eval_steps
        perplexity = torch.exp(torch.tensor(eval_loss))

        results["eval_loss"] = eval_loss
        results["perplexity"] = perplexity

        output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
        with open(output_eval_file, "w") as writer:
            for key in sorted(results.keys()):
                writer.write("{} = {}\n".format(key, str(results[key])))

        return results

        Evaluates the model on eval_dataset.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        
Evaluates the model on eval_dataset.
Utility function to be used by the eval_model() method. Not intended to be used directly.
Evaluates the model on eval_dataset . Utility function to be used by the eval_model () method . Not intended to be used directly .
def evaluate(self, eval_dataset, output_dir, multi_label=False, prefix="", verbose=True, silent=False, **kwargs):
        model = self.model
        args = self.args
        eval_output_dir = output_dir
        tokenizer = self.tokenizer
        results = {}
        def collate(examples: List[torch.Tensor]):
            if tokenizer._pad_token is None:
                return pad_sequence(examples, batch_first=True)
            return pad_sequence(examples, batch_first=True, padding_value=tokenizer.pad_token_id)
        eval_sampler = SequentialSampler(eval_dataset)
        eval_dataloader = DataLoader(
            eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate
        )
        if args.n_gpu > 1:
            model = torch.nn.DataParallel(model)
        eval_loss = 0.0
        nb_eval_steps = 0
        model.eval()
        for batch in tqdm(eval_dataloader, disable=args.silent or silent, desc="Running Evaluation"):
            inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch)
            inputs = inputs.to(self.device)
            labels = labels.to(self.device)
            with torch.no_grad():
                outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model(inputs, labels=labels)
                if args.model_type == "electra":
                    g_loss = outputs[0]
                    d_loss = outputs[1]
                    lm_loss = g_loss + args.discriminator_loss_weight * d_loss
                else:
                    lm_loss = outputs[0]
                eval_loss += lm_loss.mean().item()
            nb_eval_steps += 1
        eval_loss = eval_loss / nb_eval_steps
        perplexity = torch.exp(torch.tensor(eval_loss))
        results["eval_loss"] = eval_loss
        results["perplexity"] = perplexity
        output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
        with open(output_eval_file, "w") as writer:
            for key in sorted(results.keys()):
                writer.write("{} = {}\n".format(key, str(results[key])))
        return results
def evaluate ( self , eval_dataset , output_dir , multi_label = False , prefix = "" , verbose = True , silent = False , ** kwargs ) : model = self . model args = self . args eval_output_dir = output_dir tokenizer = self . tokenizer results = { } def collate ( examples : List [ torch . Tensor ] ) : if tokenizer . _pad_token is None : return pad_sequence ( examples , batch_first = True ) return pad_sequence ( examples , batch_first = True , padding_value = tokenizer . pad_token_id ) eval_sampler = SequentialSampler ( eval_dataset ) eval_dataloader = DataLoader ( eval_dataset , sampler = eval_sampler , batch_size = args . eval_batch_size , collate_fn = collate ) if args . n_gpu > 1 : model = torch . nn . DataParallel ( model ) eval_loss = 0.0 nb_eval_steps = 0 model . eval ( ) for batch in tqdm ( eval_dataloader , disable = args . silent or silent , desc = "Running Evaluation" ) : inputs , labels = mask_tokens ( batch , tokenizer , args ) if args . mlm else ( batch , batch ) inputs = inputs . to ( self . device ) labels = labels . to ( self . device ) with torch . no_grad ( ) : outputs = model ( inputs , masked_lm_labels = labels ) if args . mlm else model ( inputs , labels = labels ) if args . model_type == "electra" : g_loss = outputs [ 0 ] d_loss = outputs [ 1 ] lm_loss = g_loss + args . discriminator_loss_weight * d_loss else : lm_loss = outputs [ 0 ] eval_loss += lm_loss . mean ( ) . item ( ) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps perplexity = torch . exp ( torch . tensor ( eval_loss ) ) results [ "eval_loss" ] = eval_loss results [ "perplexity" ] = perplexity output_eval_file = os . path . join ( eval_output_dir , "eval_results.txt" ) with open ( output_eval_file , "w" ) as writer : for key in sorted ( results . keys ( ) ) : writer . write ( "{} = {}\n" . format ( key , str ( results [ key ] ) ) ) return results
Evaluates the model on eval_dataset.
Evaluates the model on eval_dataset .
"""
        Evaluates the model on eval_dataset.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        """
self
eval_dataset
output_dir
multi_label
prefix
verbose
silent