validator.py
def evaluate(self, disable_anneal=False, verbose=False):
        """A method that runs the validation of the model on the dataset.

        Evaluate the performance of the model on the passed dataset using the
        metrics and objectives in the Validator object.

        Args:
            verbose: A bool value determining whether we print to stdout.
                default = True.
            disable_anneal: A bool signifying whether to use 0 annealing value
                in some loss functions, default=False.

        Returns:
            A tuple consiting of a list of results of the metric evaluation and
            a list of results of the objective evaluation
        Raises:
            TypeError: If disable_anneal or verbose are not of type bool.
        """
        if not isinstance(disable_anneal, bool):
            raise TypeError('Argument: disable_anneal must be a bool.')
        if not isinstance(verbose, bool):
            raise TypeError('Argument: verbose must be a bool.')
        # Set cuda if available
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self._model.to(device)
        self._model.eval()
        # Initialise statistics if applicable
        if self._metrics:
            self._stats_metrics = np.zeros_like(self._metrics, dtype=float)
        if self._objectives:
            self._stats_objectives = np.zeros_like(self._objectives, dtype=float)
        cnt = 0
        with torch.no_grad():
            for batch in self._dataloader:
                x, y = batch
                x = Variable(x).to(device)
                y = Variable(y).to(device)
                # Some models may return more than just their predictions.
                # For these models the first element of the returned tuple
                # should contain the predictions. These models are:
                # MultiVAE,  (add to list as it grows, and modify code)
                model_output = self._model(x)
                if isinstance(self._model, MultiVAE)\
                   and isinstance(model_output, tuple):
                    # Filtering out already chosen items:
                    # In order to make sure we are not recommending already
                    # chosen items, here we set the prediction of the already
                    # chosen items to the minimum value. We know what items
                    # have already been chosen since they are in our input X.
                    model_output[0][x == 1] = torch.min(model_output[0])
                    y_out = model_output[0]
                else:
                    y_out = model_output
                    # Filtering out already chosen elements
                    y_out[x == 1] = torch.min(y_out)
                cnt += 1
                # Compute moving average with the new batch for each metric
                if self._metrics:
                    self._stats_metrics = self._stats_metrics\
                     * ((cnt - 1)/cnt)\
                     + np.array([metric.evaluate(y, y_out)
                                 for metric in self._metrics]) * (1/cnt)
                # Compute moving average with the new batch for each objective
                if self._objectives:
                    if disable_anneal:
                        self._stats_objectives = self._stats_objectives\
                         * ((cnt - 1)/cnt)\
                         + np.array([objective.compute_loss(y, model_output,
                                                            0).item()
                                    for objective in self._objectives])\
                         * (1/cnt)
                    else:
                        self._stats_objectives = self._stats_objectives\
                         * ((cnt - 1)/cnt)\
                         + np.array([objective.compute_loss(y,
                                    model_output).item()
                                    for objective in self._objectives])\
                         * (1/cnt)
        # Logging and printing of results
        if self._objectives:
            log_str = 'Validation loss:' + ''.join([' {}: {:.2f}'.format(
              self._objectives[i].name, self._stats_objectives[i])
              for i in range(len(self._objectives))])
            logging.info(log_str)
            if verbose:
                print('='*90)
                print(log_str)
        if self._metrics:
            log_str = 'Validation metrics:' + ''.join([' {}: {:.2f}'.format(
              self._metrics[i].get_name(), self._stats_metrics[i])
              for i in range(len(self._metrics))])
            logging.info(log_str)
            if verbose:
                print('='*90)
                print(log_str)
        return self._stats_metrics.tolist(), self._stats_objectives.tolist()
A method that runs the validation of the model on the dataset.

        Evaluate the performance of the model on the passed dataset using the
        metrics and objectives in the Validator object.

        Args:
            verbose: A bool value determining whether we print to stdout.
                default = True.
            disable_anneal: A bool signifying whether to use 0 annealing value
                in some loss functions, default=False.

        Returns:
            A tuple consiting of a list of results of the metric evaluation and
            a list of results of the objective evaluation
        Raises:
            TypeError: If disable_anneal or verbose are not of type bool.
        
A method that runs the validation of the model on the dataset.
Evaluate the performance of the model on the passed dataset using the
metrics and objectives in the Validator object.
A method that runs the validation of the model on the dataset . Evaluate the performance of the model on the passed dataset using the metrics and objectives in the Validator object .
def evaluate(self, disable_anneal=False, verbose=False):
        if not isinstance(disable_anneal, bool):
            raise TypeError('Argument: disable_anneal must be a bool.')
        if not isinstance(verbose, bool):
            raise TypeError('Argument: verbose must be a bool.')
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self._model.to(device)
        self._model.eval()
        if self._metrics:
            self._stats_metrics = np.zeros_like(self._metrics, dtype=float)
        if self._objectives:
            self._stats_objectives = np.zeros_like(self._objectives, dtype=float)
        cnt = 0
        with torch.no_grad():
            for batch in self._dataloader:
                x, y = batch
                x = Variable(x).to(device)
                y = Variable(y).to(device)
                model_output = self._model(x)
                if isinstance(self._model, MultiVAE)\
                   and isinstance(model_output, tuple):
                    model_output[0][x == 1] = torch.min(model_output[0])
                    y_out = model_output[0]
                else:
                    y_out = model_output
                    y_out[x == 1] = torch.min(y_out)
                cnt += 1
                if self._metrics:
                    self._stats_metrics = self._stats_metrics\
                     * ((cnt - 1)/cnt)\
                     + np.array([metric.evaluate(y, y_out)
                                 for metric in self._metrics]) * (1/cnt)
                if self._objectives:
                    if disable_anneal:
                        self._stats_objectives = self._stats_objectives\
                         * ((cnt - 1)/cnt)\
                         + np.array([objective.compute_loss(y, model_output,
                                                            0).item()
                                    for objective in self._objectives])\
                         * (1/cnt)
                    else:
                        self._stats_objectives = self._stats_objectives\
                         * ((cnt - 1)/cnt)\
                         + np.array([objective.compute_loss(y,
                                    model_output).item()
                                    for objective in self._objectives])\
                         * (1/cnt)
        if self._objectives:
            log_str = 'Validation loss:' + ''.join([' {}: {:.2f}'.format(
              self._objectives[i].name, self._stats_objectives[i])
              for i in range(len(self._objectives))])
            logging.info(log_str)
            if verbose:
                print('='*90)
                print(log_str)
        if self._metrics:
            log_str = 'Validation metrics:' + ''.join([' {}: {:.2f}'.format(
              self._metrics[i].get_name(), self._stats_metrics[i])
              for i in range(len(self._metrics))])
            logging.info(log_str)
            if verbose:
                print('='*90)
                print(log_str)
        return self._stats_metrics.tolist(), self._stats_objectives.tolist()
def evaluate ( self , disable_anneal = False , verbose = False ) : if not isinstance ( disable_anneal , bool ) : raise TypeError ( 'Argument: disable_anneal must be a bool.' ) if not isinstance ( verbose , bool ) : raise TypeError ( 'Argument: verbose must be a bool.' ) device = torch . device ( 'cuda' if torch . cuda . is_available ( ) else 'cpu' ) self . _model . to ( device ) self . _model . eval ( ) if self . _metrics : self . _stats_metrics = np . zeros_like ( self . _metrics , dtype = float ) if self . _objectives : self . _stats_objectives = np . zeros_like ( self . _objectives , dtype = float ) cnt = 0 with torch . no_grad ( ) : for batch in self . _dataloader : x , y = batch x = Variable ( x ) . to ( device ) y = Variable ( y ) . to ( device ) model_output = self . _model ( x ) if isinstance ( self . _model , MultiVAE ) and isinstance ( model_output , tuple ) : model_output [ 0 ] [ x == 1 ] = torch . min ( model_output [ 0 ] ) y_out = model_output [ 0 ] else : y_out = model_output y_out [ x == 1 ] = torch . min ( y_out ) cnt += 1 if self . _metrics : self . _stats_metrics = self . _stats_metrics * ( ( cnt - 1 ) / cnt ) + np . array ( [ metric . evaluate ( y , y_out ) for metric in self . _metrics ] ) * ( 1 / cnt ) if self . _objectives : if disable_anneal : self . _stats_objectives = self . _stats_objectives * ( ( cnt - 1 ) / cnt ) + np . array ( [ objective . compute_loss ( y , model_output , 0 ) . item ( ) for objective in self . _objectives ] ) * ( 1 / cnt ) else : self . _stats_objectives = self . _stats_objectives * ( ( cnt - 1 ) / cnt ) + np . array ( [ objective . compute_loss ( y , model_output ) . item ( ) for objective in self . _objectives ] ) * ( 1 / cnt ) if self . _objectives : log_str = 'Validation loss:' + '' . join ( [ ' {}: {:.2f}' . format ( self . _objectives [ i ] . name , self . _stats_objectives [ i ] ) for i in range ( len ( self . _objectives ) ) ] ) logging . info ( log_str ) if verbose : print ( '=' * 90 ) print ( log_str ) if self . _metrics : log_str = 'Validation metrics:' + '' . join ( [ ' {}: {:.2f}' . format ( self . _metrics [ i ] . get_name ( ) , self . _stats_metrics [ i ] ) for i in range ( len ( self . _metrics ) ) ] ) logging . info ( log_str ) if verbose : print ( '=' * 90 ) print ( log_str ) return self . _stats_metrics . tolist ( ) , self . _stats_objectives . tolist ( )
A method that runs the validation of the model on the dataset.
A method that runs the validation of the model on the dataset .
"""A method that runs the validation of the model on the dataset.

        Evaluate the performance of the model on the passed dataset using the
        metrics and objectives in the Validator object.

        Args:
            verbose: A bool value determining whether we print to stdout.
                default = True.
            disable_anneal: A bool signifying whether to use 0 annealing value
                in some loss functions, default=False.

        Returns:
            A tuple consiting of a list of results of the metric evaluation and
            a list of results of the objective evaluation
        Raises:
            TypeError: If disable_anneal or verbose are not of type bool.
        """ # Set cuda if available # Initialise statistics if applicable # Some models may return more than just their predictions. # For these models the first element of the returned tuple # should contain the predictions. These models are: # MultiVAE,  (add to list as it grows, and modify code) # Filtering out already chosen items: # In order to make sure we are not recommending already # chosen items, here we set the prediction of the already # chosen items to the minimum value. We know what items # have already been chosen since they are in our input X. # Filtering out already chosen elements # Compute moving average with the new batch for each metric # Compute moving average with the new batch for each objective # Logging and printing of results
A tuple consiting of a list of results of the metric evaluation and
a list of results of the objective evaluation
A tuple consiting of a list of results of the metric evaluation and a list of results of the objective evaluation
If disable_anneal or verbose are not of type bool.
If disable_anneal or verbose are not of type bool .
TypeError
self
disable_anneal
A bool signifying whether to use 0 annealing value
in some loss functions, default=False.
A bool signifying whether to use 0 annealing value in some loss functions default = False .
verbose
A bool value determining whether we print to stdout.
default = True.
A bool value determining whether we print to stdout . default = True .