algo/transformers/run_model.py
def compute_metrics(self, preds, labels, eval_examples, multi_label=False, **kwargs):
        """
        Computes the evaluation metrics for the model predictions.

        Args:
            preds: Model predictions
            labels: Ground truth labels
            eval_examples: List of examples on which evaluation was performed
            **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). E.g. f1=sklearn.metrics.f1_score.
                        A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions.

        Returns:
            result: Dictionary containing evaluation results. (Matthews correlation coefficient, tp, tn, fp, fn)
            wrong: List of InputExample objects corresponding to each incorrect prediction by the model
        """  # noqa: ignore flake8"

        assert len(preds) == len(labels)

        extra_metrics = {}
        for metric, func in kwargs.items():
            extra_metrics[metric] = func(labels, preds)

        mismatched = labels != preds

        wrong = [i for (i, v) in zip(eval_examples, mismatched) if v.any()]

        if multi_label:
            label_ranking_score = label_ranking_average_precision_score(labels, preds)
            return {**{"LRAP": label_ranking_score}, **extra_metrics}, wrong
        elif self.args["regression"]:
            return {**extra_metrics}, wrong

        mcc = matthews_corrcoef(labels, preds)

        if self.model.num_labels == 2:
            tn, fp, fn, tp = confusion_matrix(labels, preds).ravel()
            return (
                {**{"mcc": mcc, "tp": tp, "tn": tn, "fp": fp, "fn": fn}, **extra_metrics},
                wrong,
            )
        else:
            return {**{"mcc": mcc}, **extra_metrics}, wrong

        Computes the evaluation metrics for the model predictions.

        Args:
            preds: Model predictions
            labels: Ground truth labels
            eval_examples: List of examples on which evaluation was performed
            **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). E.g. f1=sklearn.metrics.f1_score.
                        A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions.

        Returns:
            result: Dictionary containing evaluation results. (Matthews correlation coefficient, tp, tn, fp, fn)
            wrong: List of InputExample objects corresponding to each incorrect prediction by the model
        
Computes the evaluation metrics for the model predictions.
Computes the evaluation metrics for the model predictions .
def compute_metrics(self, preds, labels, eval_examples, multi_label=False, **kwargs):
        assert len(preds) == len(labels)
        extra_metrics = {}
        for metric, func in kwargs.items():
            extra_metrics[metric] = func(labels, preds)
        mismatched = labels != preds
        wrong = [i for (i, v) in zip(eval_examples, mismatched) if v.any()]
        if multi_label:
            label_ranking_score = label_ranking_average_precision_score(labels, preds)
            return {**{"LRAP": label_ranking_score}, **extra_metrics}, wrong
        elif self.args["regression"]:
            return {**extra_metrics}, wrong
        mcc = matthews_corrcoef(labels, preds)
        if self.model.num_labels == 2:
            tn, fp, fn, tp = confusion_matrix(labels, preds).ravel()
            return (
                {**{"mcc": mcc, "tp": tp, "tn": tn, "fp": fp, "fn": fn}, **extra_metrics},
                wrong,
            )
        else:
            return {**{"mcc": mcc}, **extra_metrics}, wrong
def compute_metrics ( self , preds , labels , eval_examples , multi_label = False , ** kwargs ) : assert len ( preds ) == len ( labels ) extra_metrics = { } for metric , func in kwargs . items ( ) : extra_metrics [ metric ] = func ( labels , preds ) mismatched = labels != preds wrong = [ i for ( i , v ) in zip ( eval_examples , mismatched ) if v . any ( ) ] if multi_label : label_ranking_score = label_ranking_average_precision_score ( labels , preds ) return { ** { "LRAP" : label_ranking_score } , ** extra_metrics } , wrong elif self . args [ "regression" ] : return { ** extra_metrics } , wrong mcc = matthews_corrcoef ( labels , preds ) if self . model . num_labels == 2 : tn , fp , fn , tp = confusion_matrix ( labels , preds ) . ravel ( ) return ( { ** { "mcc" : mcc , "tp" : tp , "tn" : tn , "fp" : fp , "fn" : fn } , ** extra_metrics } , wrong , ) else : return { ** { "mcc" : mcc } , ** extra_metrics } , wrong
Computes the evaluation metrics for the model predictions.
Computes the evaluation metrics for the model predictions .
"""
        Computes the evaluation metrics for the model predictions.

        Args:
            preds: Model predictions
            labels: Ground truth labels
            eval_examples: List of examples on which evaluation was performed
            **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). E.g. f1=sklearn.metrics.f1_score.
                        A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions.

        Returns:
            result: Dictionary containing evaluation results. (Matthews correlation coefficient, tp, tn, fp, fn)
            wrong: List of InputExample objects corresponding to each incorrect prediction by the model
        """ # noqa: ignore flake8"
Dictionary containing evaluation results. (Matthews correlation coefficient, tp, tn, fp, fn)
wrong: List of InputExample objects corresponding to each incorrect prediction by the model
Dictionary containing evaluation results . ( Matthews correlation coefficient tp tn fp fn ) wrong : List of InputExample objects corresponding to each incorrect prediction by the model
result
self
preds
None
labels
Ground truth labels
Ground truth labels
eval_examples
List of examples on which evaluation was performed
List of examples on which evaluation was performed
multi_label
**kwargs
Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). E.g. f1=sklearn.metrics.f1_score.
A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions.
Additional metrics that should be used . Pass in the metrics as keyword arguments ( name of metric : function to use ) . E . g . f1 = sklearn . metrics . f1_score . A metric function should take in two parameters . The first parameter will be the true labels and the second parameter will be the predictions .