algo/transformers/run_model.py
def evaluate(self, eval_df, output_dir, multi_label=False, prefix="", verbose=True, silent=False, **kwargs):
        """
        Evaluates the model on eval_df.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        """

        device = self.device
        model = self.model
        args = self.args
        eval_output_dir = output_dir

        results = {}

        if "text" in eval_df.columns and "labels" in eval_df.columns:
            eval_examples = [
                InputExample(i, text, None, label)
                for i, (text, label) in enumerate(zip(eval_df["text"], eval_df["labels"]))
            ]
        elif "text_a" in eval_df.columns and "text_b" in eval_df.columns:
            eval_examples = [
                InputExample(i, text_a, text_b, label)
                for i, (text_a, text_b, label) in enumerate(
                    zip(eval_df["text_a"], eval_df["text_b"], eval_df["labels"])
                )
            ]
        else:
            warnings.warn(
                "Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels."
            )
            eval_examples = [
                InputExample(i, text, None, label)
                for i, (text, label) in enumerate(zip(eval_df.iloc[:, 0], eval_df.iloc[:, 1]))
            ]

        if args["sliding_window"]:
            eval_dataset, window_counts = self.load_and_cache_examples(
                eval_examples, evaluate=True, verbose=verbose, silent=silent
            )
        else:
            eval_dataset = self.load_and_cache_examples(eval_examples, evaluate=True, verbose=verbose, silent=silent)
        os.makedirs(eval_output_dir, exist_ok=True)

        eval_sampler = SequentialSampler(eval_dataset)
        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])

        eval_loss = 0.0
        nb_eval_steps = 0
        preds = None
        out_label_ids = None
        model.eval()

        for batch in tqdm(eval_dataloader, disable=args["silent"] or silent):
            batch = tuple(t.to(device) for t in batch)

            with torch.no_grad():
                inputs = self._get_inputs_dict(batch)

                outputs = model(**inputs)
                tmp_eval_loss, logits = outputs[:2]

                if multi_label:
                    logits = logits.sigmoid()
                eval_loss += tmp_eval_loss.mean().item()

            nb_eval_steps += 1

            if preds is None:
                preds = logits.detach().cpu().numpy()
                out_label_ids = inputs["labels"].detach().cpu().numpy()
            else:
                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
                out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)

        eval_loss = eval_loss / nb_eval_steps

        if args["sliding_window"]:
            count = 0
            window_ranges = []
            for n_windows in window_counts:
                window_ranges.append([count, count + n_windows])
                count += n_windows

            preds = [preds[window_range[0] : window_range[1]] for window_range in window_ranges]
            out_label_ids = [
                out_label_ids[i] for i in range(len(out_label_ids)) if i in [window[0] for window in window_ranges]
            ]

            model_outputs = preds

            preds = [np.argmax(pred, axis=1) for pred in preds]
            final_preds = []
            for pred_row in preds:
                mode_pred, counts = mode(pred_row)
                if len(counts) > 1 and counts[0] == counts[1]:
                    final_preds.append(args["tie_value"])
                else:
                    final_preds.append(mode_pred[0])
            preds = np.array(final_preds)
        elif not multi_label and args["regression"] is True:
            preds = np.squeeze(preds)
            model_outputs = preds
        else:
            model_outputs = preds

            if not multi_label:
                preds = np.argmax(preds, axis=1)

        result, wrong = self.compute_metrics(preds, out_label_ids, eval_examples, **kwargs)
        result["eval_loss"] = eval_loss
        results.update(result)

        output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
        with open(output_eval_file, "w") as writer:
            for key in sorted(result.keys()):
                writer.write("{} = {}\n".format(key, str(result[key])))

        return results, model_outputs, wrong

        Evaluates the model on eval_df.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        
Evaluates the model on eval_df.
Utility function to be used by the eval_model() method. Not intended to be used directly.
Evaluates the model on eval_df . Utility function to be used by the eval_model () method . Not intended to be used directly .
def evaluate(self, eval_df, output_dir, multi_label=False, prefix="", verbose=True, silent=False, **kwargs):
        device = self.device
        model = self.model
        args = self.args
        eval_output_dir = output_dir
        results = {}
        if "text" in eval_df.columns and "labels" in eval_df.columns:
            eval_examples = [
                InputExample(i, text, None, label)
                for i, (text, label) in enumerate(zip(eval_df["text"], eval_df["labels"]))
            ]
        elif "text_a" in eval_df.columns and "text_b" in eval_df.columns:
            eval_examples = [
                InputExample(i, text_a, text_b, label)
                for i, (text_a, text_b, label) in enumerate(
                    zip(eval_df["text_a"], eval_df["text_b"], eval_df["labels"])
                )
            ]
        else:
            warnings.warn(
                "Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels."
            )
            eval_examples = [
                InputExample(i, text, None, label)
                for i, (text, label) in enumerate(zip(eval_df.iloc[:, 0], eval_df.iloc[:, 1]))
            ]
        if args["sliding_window"]:
            eval_dataset, window_counts = self.load_and_cache_examples(
                eval_examples, evaluate=True, verbose=verbose, silent=silent
            )
        else:
            eval_dataset = self.load_and_cache_examples(eval_examples, evaluate=True, verbose=verbose, silent=silent)
        os.makedirs(eval_output_dir, exist_ok=True)
        eval_sampler = SequentialSampler(eval_dataset)
        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args["eval_batch_size"])
        eval_loss = 0.0
        nb_eval_steps = 0
        preds = None
        out_label_ids = None
        model.eval()
        for batch in tqdm(eval_dataloader, disable=args["silent"] or silent):
            batch = tuple(t.to(device) for t in batch)
            with torch.no_grad():
                inputs = self._get_inputs_dict(batch)
                outputs = model(**inputs)
                tmp_eval_loss, logits = outputs[:2]
                if multi_label:
                    logits = logits.sigmoid()
                eval_loss += tmp_eval_loss.mean().item()
            nb_eval_steps += 1
            if preds is None:
                preds = logits.detach().cpu().numpy()
                out_label_ids = inputs["labels"].detach().cpu().numpy()
            else:
                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
                out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
        eval_loss = eval_loss / nb_eval_steps
        if args["sliding_window"]:
            count = 0
            window_ranges = []
            for n_windows in window_counts:
                window_ranges.append([count, count + n_windows])
                count += n_windows
            preds = [preds[window_range[0] : window_range[1]] for window_range in window_ranges]
            out_label_ids = [
                out_label_ids[i] for i in range(len(out_label_ids)) if i in [window[0] for window in window_ranges]
            ]
            model_outputs = preds
            preds = [np.argmax(pred, axis=1) for pred in preds]
            final_preds = []
            for pred_row in preds:
                mode_pred, counts = mode(pred_row)
                if len(counts) > 1 and counts[0] == counts[1]:
                    final_preds.append(args["tie_value"])
                else:
                    final_preds.append(mode_pred[0])
            preds = np.array(final_preds)
        elif not multi_label and args["regression"] is True:
            preds = np.squeeze(preds)
            model_outputs = preds
        else:
            model_outputs = preds
            if not multi_label:
                preds = np.argmax(preds, axis=1)
        result, wrong = self.compute_metrics(preds, out_label_ids, eval_examples, **kwargs)
        result["eval_loss"] = eval_loss
        results.update(result)
        output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
        with open(output_eval_file, "w") as writer:
            for key in sorted(result.keys()):
                writer.write("{} = {}\n".format(key, str(result[key])))
        return results, model_outputs, wrong
def evaluate ( self , eval_df , output_dir , multi_label = False , prefix = "" , verbose = True , silent = False , ** kwargs ) : device = self . device model = self . model args = self . args eval_output_dir = output_dir results = { } if "text" in eval_df . columns and "labels" in eval_df . columns : eval_examples = [ InputExample ( i , text , None , label ) for i , ( text , label ) in enumerate ( zip ( eval_df [ "text" ] , eval_df [ "labels" ] ) ) ] elif "text_a" in eval_df . columns and "text_b" in eval_df . columns : eval_examples = [ InputExample ( i , text_a , text_b , label ) for i , ( text_a , text_b , label ) in enumerate ( zip ( eval_df [ "text_a" ] , eval_df [ "text_b" ] , eval_df [ "labels" ] ) ) ] else : warnings . warn ( "Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels." ) eval_examples = [ InputExample ( i , text , None , label ) for i , ( text , label ) in enumerate ( zip ( eval_df . iloc [ : , 0 ] , eval_df . iloc [ : , 1 ] ) ) ] if args [ "sliding_window" ] : eval_dataset , window_counts = self . load_and_cache_examples ( eval_examples , evaluate = True , verbose = verbose , silent = silent ) else : eval_dataset = self . load_and_cache_examples ( eval_examples , evaluate = True , verbose = verbose , silent = silent ) os . makedirs ( eval_output_dir , exist_ok = True ) eval_sampler = SequentialSampler ( eval_dataset ) eval_dataloader = DataLoader ( eval_dataset , sampler = eval_sampler , batch_size = args [ "eval_batch_size" ] ) eval_loss = 0.0 nb_eval_steps = 0 preds = None out_label_ids = None model . eval ( ) for batch in tqdm ( eval_dataloader , disable = args [ "silent" ] or silent ) : batch = tuple ( t . to ( device ) for t in batch ) with torch . no_grad ( ) : inputs = self . _get_inputs_dict ( batch ) outputs = model ( ** inputs ) tmp_eval_loss , logits = outputs [ : 2 ] if multi_label : logits = logits . sigmoid ( ) eval_loss += tmp_eval_loss . mean ( ) . item ( ) nb_eval_steps += 1 if preds is None : preds = logits . detach ( ) . cpu ( ) . numpy ( ) out_label_ids = inputs [ "labels" ] . detach ( ) . cpu ( ) . numpy ( ) else : preds = np . append ( preds , logits . detach ( ) . cpu ( ) . numpy ( ) , axis = 0 ) out_label_ids = np . append ( out_label_ids , inputs [ "labels" ] . detach ( ) . cpu ( ) . numpy ( ) , axis = 0 ) eval_loss = eval_loss / nb_eval_steps if args [ "sliding_window" ] : count = 0 window_ranges = [ ] for n_windows in window_counts : window_ranges . append ( [ count , count + n_windows ] ) count += n_windows preds = [ preds [ window_range [ 0 ] : window_range [ 1 ] ] for window_range in window_ranges ] out_label_ids = [ out_label_ids [ i ] for i in range ( len ( out_label_ids ) ) if i in [ window [ 0 ] for window in window_ranges ] ] model_outputs = preds preds = [ np . argmax ( pred , axis = 1 ) for pred in preds ] final_preds = [ ] for pred_row in preds : mode_pred , counts = mode ( pred_row ) if len ( counts ) > 1 and counts [ 0 ] == counts [ 1 ] : final_preds . append ( args [ "tie_value" ] ) else : final_preds . append ( mode_pred [ 0 ] ) preds = np . array ( final_preds ) elif not multi_label and args [ "regression" ] is True : preds = np . squeeze ( preds ) model_outputs = preds else : model_outputs = preds if not multi_label : preds = np . argmax ( preds , axis = 1 ) result , wrong = self . compute_metrics ( preds , out_label_ids , eval_examples , ** kwargs ) result [ "eval_loss" ] = eval_loss results . update ( result ) output_eval_file = os . path . join ( eval_output_dir , "eval_results.txt" ) with open ( output_eval_file , "w" ) as writer : for key in sorted ( result . keys ( ) ) : writer . write ( "{} = {}\n" . format ( key , str ( result [ key ] ) ) ) return results , model_outputs , wrong
Evaluates the model on eval_df.
Evaluates the model on eval_df .
"""
        Evaluates the model on eval_df.

        Utility function to be used by the eval_model() method. Not intended to be used directly.
        """
self
eval_df
output_dir
multi_label
prefix
verbose
silent