jiant/trainer.py
def _calculate_validation_performance(
        self, task, task_infos, tasks, batch_size, all_val_metrics, n_examples_overall
    ):
        """
        Builds validation generator, evaluates on each task and produces validation metrics.

        Parameters
        ----------
        task: current task to get validation performance of
        task_infos: Instance of information about the task (see _setup_training for definition)
        tasks: list of task objects to train on
        batch_size: int, batch size to use for the tasks
        all_val_metrics: dictionary. storing the validation performance
        n_examples_overall = int, current number of examples the model is validated on

        Returns
        -------
        n_examples_overall: int, current number of examples
        task_infos: updated Instance with reset training progress
        all_val_metrics: dictionary updated with micro and macro average validation performance
        """
        n_examples, batch_num = 0, 0
        task_info = task_infos[task.name]
        # to speed up training, we evaluate on a subset of validation data
        if self._val_data_limit >= 0:
            max_data_points = min(task.n_val_examples, self._val_data_limit)
        else:
            max_data_points = task.n_val_examples
        #NOTE If we build a new generator for each validation, must see same max_data_points each time
        val_generator = BasicIterator(batch_size, instances_per_epoch=max_data_points)(
            task.val_data, num_epochs=1, shuffle=False
        )
        val_generator = move_to_device(val_generator, self._cuda_device)
        n_val_batches = math.ceil(max_data_points / batch_size)
        #NOTE Reset validation loss on each validation, so not an average
        all_val_metrics["%s_loss" % task.name] = 0.0

        for batch in val_generator:
            batch_num += 1
            with torch.no_grad():
                out = self._forward(batch, task=task)
            if not 'discriminator' in task.name:
                loss = out["loss"]
            else:
                loss = out["loss_shared"]
            all_val_metrics["%s_loss" % task.name] += loss.data.cpu().numpy()
            n_examples += out["n_exs"]

            # log
            if time.time() - task_info["last_log"] > self._log_interval:
                task_metrics = task.get_metrics()
                task_metrics["%s_loss" % task.name] = (
                    all_val_metrics["%s_loss" % task.name] / batch_num
                )
                description = self._description_from_metrics(task_metrics)
                log.info(
                    "Evaluate: task %s, batch %d (%d): %s",
                    task.name,
                    batch_num,
                    n_val_batches,
                    description,
                )
                task_info["last_log"] = time.time()
        assert batch_num == n_val_batches

        # Get task validation metrics and store in all_val_metrics
        task_metrics = task.get_metrics(reset=True)
        for name, value in task_metrics.items():
            all_val_metrics["%s_%s" % (task.name, name)] = value
        all_val_metrics["%s_loss" % task.name] /= batch_num  # n_val_batches
        if task.val_metric_decreases and len(tasks) > 1:
            all_val_metrics["micro_avg"] += (
                1 - all_val_metrics[task.val_metric] / self._dec_val_scale
            ) * n_examples
            if not 'adv' in task.name:
                all_val_metrics["macro_avg"] += (
                    1 - all_val_metrics[task.val_metric] / self._dec_val_scale
                ) #XXX
        else:
            # triggers for single-task cases and during MTL when task val metric increases
            all_val_metrics["micro_avg"] += all_val_metrics[task.val_metric] * n_examples
            if not 'adv' in task.name:
                all_val_metrics["macro_avg"] += all_val_metrics[task.val_metric] #XXX
        n_examples_overall += n_examples

        # Reset training progress
        task_info["n_batches_since_val"] = 0
        task_info["loss"] = 0 #NOTE this is only ever used for training loss
        all_val_metrics["micro_avg"] /= n_examples_overall
        log.info(f'Macro avg: {all_val_metrics["macro_avg"]} being divided by len(tasks): {len(tasks)}')
        if not 'adv' in task.name:
            all_val_metrics["macro_avg"] /= len([task for task in tasks if not 'adv' in task.name]) #XXX we divide by the number of tasks every time??
        return n_examples_overall, task_infos, all_val_metrics

        Builds validation generator, evaluates on each task and produces validation metrics.

        Parameters
        ----------
        task: current task to get validation performance of
        task_infos: Instance of information about the task (see _setup_training for definition)
        tasks: list of task objects to train on
        batch_size: int, batch size to use for the tasks
        all_val_metrics: dictionary. storing the validation performance
        n_examples_overall = int, current number of examples the model is validated on

        Returns
        -------
        n_examples_overall: int, current number of examples
        task_infos: updated Instance with reset training progress
        all_val_metrics: dictionary updated with micro and macro average validation performance
        
Builds validation generator, evaluates on each task and produces validation metrics.
Parameters

current task to get validation performance of
task_infos: Instance of information about the task
tasks: list of task objects to train on
batch_size: int, batch size to use for the tasks
all_val_metrics: dictionary. storing the validation performance
n_examples_overall = int, current number of examples the model is validated on

Returns

int, current number of examples
task_infos: updated Instance with reset training progress
all_val_metrics: dictionary updated with micro and macro average validation performance
Builds validation generator evaluates on each task and produces validation metrics . Parameters current task to get validation performance of task_infos : Instance of information about the task tasks : list of task objects to train on batch_size : int batch size to use for the tasks all_val_metrics : dictionary . storing the validation performance n_examples_overall = int current number of examples the model is validated on Returns int current number of examples task_infos : updated Instance with reset training progress all_val_metrics : dictionary updated with micro and macro average validation performance
def _calculate_validation_performance(
        self, task, task_infos, tasks, batch_size, all_val_metrics, n_examples_overall
    ):
        n_examples, batch_num = 0, 0
        task_info = task_infos[task.name]
        if self._val_data_limit >= 0:
            max_data_points = min(task.n_val_examples, self._val_data_limit)
        else:
            max_data_points = task.n_val_examples
        val_generator = BasicIterator(batch_size, instances_per_epoch=max_data_points)(
            task.val_data, num_epochs=1, shuffle=False
        )
        val_generator = move_to_device(val_generator, self._cuda_device)
        n_val_batches = math.ceil(max_data_points / batch_size)
        all_val_metrics["%s_loss" % task.name] = 0.0
        for batch in val_generator:
            batch_num += 1
            with torch.no_grad():
                out = self._forward(batch, task=task)
            if not 'discriminator' in task.name:
                loss = out["loss"]
            else:
                loss = out["loss_shared"]
            all_val_metrics["%s_loss" % task.name] += loss.data.cpu().numpy()
            n_examples += out["n_exs"]
            if time.time() - task_info["last_log"] > self._log_interval:
                task_metrics = task.get_metrics()
                task_metrics["%s_loss" % task.name] = (
                    all_val_metrics["%s_loss" % task.name] / batch_num
                )
                description = self._description_from_metrics(task_metrics)
                log.info(
                    "Evaluate: task %s, batch %d (%d): %s",
                    task.name,
                    batch_num,
                    n_val_batches,
                    description,
                )
                task_info["last_log"] = time.time()
        assert batch_num == n_val_batches
        task_metrics = task.get_metrics(reset=True)
        for name, value in task_metrics.items():
            all_val_metrics["%s_%s" % (task.name, name)] = value
        all_val_metrics["%s_loss" % task.name] /= batch_num  
        if task.val_metric_decreases and len(tasks) > 1:
            all_val_metrics["micro_avg"] += (
                1 - all_val_metrics[task.val_metric] / self._dec_val_scale
            ) * n_examples
            if not 'adv' in task.name:
                all_val_metrics["macro_avg"] += (
                    1 - all_val_metrics[task.val_metric] / self._dec_val_scale
                ) 
        else:
            all_val_metrics["micro_avg"] += all_val_metrics[task.val_metric] * n_examples
            if not 'adv' in task.name:
                all_val_metrics["macro_avg"] += all_val_metrics[task.val_metric] 
        n_examples_overall += n_examples
        task_info["n_batches_since_val"] = 0
        task_info["loss"] = 0 
        all_val_metrics["micro_avg"] /= n_examples_overall
        log.info(f'Macro avg: {all_val_metrics["macro_avg"]} being divided by len(tasks): {len(tasks)}')
        if not 'adv' in task.name:
            all_val_metrics["macro_avg"] /= len([task for task in tasks if not 'adv' in task.name])  we divide by the number of tasks every time??
        return n_examples_overall, task_infos, all_val_metrics
def _calculate_validation_performance ( self , task , task_infos , tasks , batch_size , all_val_metrics , n_examples_overall ) : n_examples , batch_num = 0 , 0 task_info = task_infos [ task . name ] if self . _val_data_limit >= 0 : max_data_points = min ( task . n_val_examples , self . _val_data_limit ) else : max_data_points = task . n_val_examples val_generator = BasicIterator ( batch_size , instances_per_epoch = max_data_points ) ( task . val_data , num_epochs = 1 , shuffle = False ) val_generator = move_to_device ( val_generator , self . _cuda_device ) n_val_batches = math . ceil ( max_data_points / batch_size ) all_val_metrics [ "%s_loss" % task . name ] = 0.0 for batch in val_generator : batch_num += 1 with torch . no_grad ( ) : out = self . _forward ( batch , task = task ) if not 'discriminator' in task . name : loss = out [ "loss" ] else : loss = out [ "loss_shared" ] all_val_metrics [ "%s_loss" % task . name ] += loss . data . cpu ( ) . numpy ( ) n_examples += out [ "n_exs" ] if time . time ( ) - task_info [ "last_log" ] > self . _log_interval : task_metrics = task . get_metrics ( ) task_metrics [ "%s_loss" % task . name ] = ( all_val_metrics [ "%s_loss" % task . name ] / batch_num ) description = self . _description_from_metrics ( task_metrics ) log . info ( "Evaluate: task %s, batch %d (%d): %s" , task . name , batch_num , n_val_batches , description , ) task_info [ "last_log" ] = time . time ( ) assert batch_num == n_val_batches task_metrics = task . get_metrics ( reset = True ) for name , value in task_metrics . items ( ) : all_val_metrics [ "%s_%s" % ( task . name , name ) ] = value all_val_metrics [ "%s_loss" % task . name ] /= batch_num if task . val_metric_decreases and len ( tasks ) > 1 : all_val_metrics [ "micro_avg" ] += ( 1 - all_val_metrics [ task . val_metric ] / self . _dec_val_scale ) * n_examples if not 'adv' in task . name : all_val_metrics [ "macro_avg" ] += ( 1 - all_val_metrics [ task . val_metric ] / self . _dec_val_scale ) else : all_val_metrics [ "micro_avg" ] += all_val_metrics [ task . val_metric ] * n_examples if not 'adv' in task . name : all_val_metrics [ "macro_avg" ] += all_val_metrics [ task . val_metric ] n_examples_overall += n_examples task_info [ "n_batches_since_val" ] = 0 task_info [ "loss" ] = 0 all_val_metrics [ "micro_avg" ] /= n_examples_overall log . info ( f'Macro avg: {all_val_metrics["macro_avg"]} being divided by len(tasks): {len(tasks)}' ) if not 'adv' in task . name : all_val_metrics [ "macro_avg" ] /= len ( [ task for task in tasks if not 'adv' in task . name ] ) return n_examples_overall , task_infos , all_val_metrics
Builds validation generator, evaluates on each task and produces validation metrics.
Builds validation generator evaluates on each task and produces validation metrics .
"""
        Builds validation generator, evaluates on each task and produces validation metrics.

        Parameters
        ----------
        task: current task to get validation performance of
        task_infos: Instance of information about the task (see _setup_training for definition)
        tasks: list of task objects to train on
        batch_size: int, batch size to use for the tasks
        all_val_metrics: dictionary. storing the validation performance
        n_examples_overall = int, current number of examples the model is validated on

        Returns
        -------
        n_examples_overall: int, current number of examples
        task_infos: updated Instance with reset training progress
        all_val_metrics: dictionary updated with micro and macro average validation performance
        """ # to speed up training, we evaluate on a subset of validation data #NOTE If we build a new generator for each validation, must see same max_data_points each time #NOTE Reset validation loss on each validation, so not an average # log # Get task validation metrics and store in all_val_metrics # n_val_batches #XXX # triggers for single-task cases and during MTL when task val metric increases #XXX # Reset training progress #NOTE this is only ever used for training loss #XXX we divide by the number of tasks every time??
self
task
task_infos
tasks
batch_size
all_val_metrics
n_examples_overall