jiant/trainer.py
def _setup_training(
        self, tasks, batch_size, train_params, optimizer_params, scheduler_params, phase
    ):
        """ Set up the trainer by initializing task_infos and metric_infos, which
        track necessary information about the training status of each task and metric respectively.

        Returns:
            - task_infos (Dict[str:Dict[str:???]]): dictionary containing where each task_info
              contains:
                - iterator: a task specific (because it uses that task's fields to dynamically
                    batch) batcher
                - n_tr_batches: the number of training batches
                - tr_generator: generator object that returns the batches, set to repeat
                    indefinitely
                - loss: the accumulated loss (during training or validation)
                - n_batches_since_val: number of batches trained on since the last validation
                - total_batches_trained: number of batches trained over all validation checks
                - optimizer: a task specific optimizer, not used if the global optimizer is not
                    None
                - scheduler: a task specific scheduler, not used if the global optimizer is not
                    None
                - stopped: a bool indicating if that task is stopped or not (if it ran out of
                    patience or hit min lr)
                - last_log: the time we last logged progress for the task

            - metric_infos (Dict[str:Dict[str:???]]): dictionary containing metric information.
                Each metric should be the validation metric of a task, except {micro/macro}_avg,
                which are privileged to get an aggregate multi-task score. Each dict contains:
                - hist (List[float]): previous values of the metric
                - stopped (Bool): whether or not that metric is stopped or not
                - best (Tuple(Int, Dict)): information on the best value of that metric and when
                    it happened
        """
        task_infos = {task.name: {} for task in tasks}
        for task in tasks:
            task_info = task_infos[task.name]
            if (
                os.path.exists(os.path.join(self._serialization_dir, task.name)) is False
                and phase == "target_train"
            ):
                os.mkdir(os.path.join(self._serialization_dir, task.name))

            # Adding task-specific smart iterator to speed up training
            instance = [i for i in itertools.islice(task.train_data, 1)][0]
            pad_dict = instance.get_padding_lengths()
            sorting_keys = []
            for field in pad_dict:
                for pad_field in pad_dict[field]:
                    sorting_keys.append((field, pad_field))
            iterator = BucketIterator(
                sorting_keys=sorting_keys,
                max_instances_in_memory=10000,
                batch_size=batch_size,
                biggest_batch_first=True,
            )
            tr_generator = iterator(task.train_data, num_epochs=None)
            tr_generator = move_to_device(tr_generator, self._cuda_device)
            task_info["iterator"] = iterator

            if phase == "pretrain":
                # Warning: This won't be precise when training_data_fraction is set, since each
                #  example is included or excluded independently using a hashing function.
                # Fortunately, it doesn't need to be.
                log.info(f'Task {task.name} has pretrain fraction {task.pretrain_fraction}')
                task_info["n_tr_batches"] = math.ceil(
                    #task.n_train_examples * self._training_data_fraction / batch_size # Original
                    task.n_train_examples * task.pretrain_fraction / batch_size #XXX
                )
                log.info(f'Task {task.name} will have {math.ceil(task.n_train_examples * task.pretrain_fraction)} many training examples after subsampling.')
            else:
                task_info["n_tr_batches"] = math.ceil(task.n_train_examples / batch_size)

            task_info["tr_generator"] = tr_generator
            task_info["loss"] = 0.0
            task_info["total_batches_trained"] = 0
            task_info["n_batches_since_val"] = 0
            # deepcopy b/c using Params pops values and we may want to reuse
            # the Params object later
            opt_params = copy.deepcopy(optimizer_params)
            if self._max_epochs > 0 and "t_total" in optimizer_params:
                # If we know in advance how many opt steps for the transformer
                # there are, set it.
                opt_params["t_total"] = task_info["n_tr_batches"] * self._max_epochs
            task_info["optimizer"] = Optimizer.from_params(train_params, opt_params)
            task_info["scheduler"] = LearningRateScheduler.from_params(
                task_info["optimizer"], copy.deepcopy(scheduler_params)
            )
            task_info["stopped"] = False
            task_info["last_log"] = time.time()

        # Metric bookkeeping
        all_metrics = [task.val_metric for task in tasks] + ["micro_avg", "macro_avg"]
        metric_infos = {
            metric: {"hist": [], "stopped": False, "best": (-1, {})} for metric in all_metrics
        }
        self.task_to_metric_mapping = {task.val_metric: task.name for task in tasks}
        self._task_infos = task_infos
        self._metric_infos = metric_infos

        return task_infos, metric_infos
 Set up the trainer by initializing task_infos and metric_infos, which
        track necessary information about the training status of each task and metric respectively.

        Returns:
            - task_infos (Dict[str:Dict[str:???]]): dictionary containing where each task_info
              contains:
                - iterator: a task specific (because it uses that task's fields to dynamically
                    batch) batcher
                - n_tr_batches: the number of training batches
                - tr_generator: generator object that returns the batches, set to repeat
                    indefinitely
                - loss: the accumulated loss (during training or validation)
                - n_batches_since_val: number of batches trained on since the last validation
                - total_batches_trained: number of batches trained over all validation checks
                - optimizer: a task specific optimizer, not used if the global optimizer is not
                    None
                - scheduler: a task specific scheduler, not used if the global optimizer is not
                    None
                - stopped: a bool indicating if that task is stopped or not (if it ran out of
                    patience or hit min lr)
                - last_log: the time we last logged progress for the task

            - metric_infos (Dict[str:Dict[str:???]]): dictionary containing metric information.
                Each metric should be the validation metric of a task, except {micro/macro}_avg,
                which are privileged to get an aggregate multi-task score. Each dict contains:
                - hist (List[float]): previous values of the metric
                - stopped (Bool): whether or not that metric is stopped or not
                - best (Tuple(Int, Dict)): information on the best value of that metric and when
                    it happened
        
Set up the trainer by initializing task_infos and metric_infos, which
track necessary information about the training status of each task and metric respectively.
Set up the trainer by initializing task_infos and metric_infos which track necessary information about the training status of each task and metric respectively .
def _setup_training(
        self, tasks, batch_size, train_params, optimizer_params, scheduler_params, phase
    ):
        task_infos = {task.name: {} for task in tasks}
        for task in tasks:
            task_info = task_infos[task.name]
            if (
                os.path.exists(os.path.join(self._serialization_dir, task.name)) is False
                and phase == "target_train"
            ):
                os.mkdir(os.path.join(self._serialization_dir, task.name))
            instance = [i for i in itertools.islice(task.train_data, 1)][0]
            pad_dict = instance.get_padding_lengths()
            sorting_keys = []
            for field in pad_dict:
                for pad_field in pad_dict[field]:
                    sorting_keys.append((field, pad_field))
            iterator = BucketIterator(
                sorting_keys=sorting_keys,
                max_instances_in_memory=10000,
                batch_size=batch_size,
                biggest_batch_first=True,
            )
            tr_generator = iterator(task.train_data, num_epochs=None)
            tr_generator = move_to_device(tr_generator, self._cuda_device)
            task_info["iterator"] = iterator
            if phase == "pretrain":
                log.info(f'Task {task.name} has pretrain fraction {task.pretrain_fraction}')
                task_info["n_tr_batches"] = math.ceil(
                    task.n_train_examples * task.pretrain_fraction / batch_size 
                )
                log.info(f'Task {task.name} will have {math.ceil(task.n_train_examples * task.pretrain_fraction)} many training examples after subsampling.')
            else:
                task_info["n_tr_batches"] = math.ceil(task.n_train_examples / batch_size)
            task_info["tr_generator"] = tr_generator
            task_info["loss"] = 0.0
            task_info["total_batches_trained"] = 0
            task_info["n_batches_since_val"] = 0
            opt_params = copy.deepcopy(optimizer_params)
            if self._max_epochs > 0 and "t_total" in optimizer_params:
                opt_params["t_total"] = task_info["n_tr_batches"] * self._max_epochs
            task_info["optimizer"] = Optimizer.from_params(train_params, opt_params)
            task_info["scheduler"] = LearningRateScheduler.from_params(
                task_info["optimizer"], copy.deepcopy(scheduler_params)
            )
            task_info["stopped"] = False
            task_info["last_log"] = time.time()
        all_metrics = [task.val_metric for task in tasks] + ["micro_avg", "macro_avg"]
        metric_infos = {
            metric: {"hist": [], "stopped": False, "best": (-1, {})} for metric in all_metrics
        }
        self.task_to_metric_mapping = {task.val_metric: task.name for task in tasks}
        self._task_infos = task_infos
        self._metric_infos = metric_infos
        return task_infos, metric_infos
def _setup_training ( self , tasks , batch_size , train_params , optimizer_params , scheduler_params , phase ) : task_infos = { task . name : { } for task in tasks } for task in tasks : task_info = task_infos [ task . name ] if ( os . path . exists ( os . path . join ( self . _serialization_dir , task . name ) ) is False and phase == "target_train" ) : os . mkdir ( os . path . join ( self . _serialization_dir , task . name ) ) instance = [ i for i in itertools . islice ( task . train_data , 1 ) ] [ 0 ] pad_dict = instance . get_padding_lengths ( ) sorting_keys = [ ] for field in pad_dict : for pad_field in pad_dict [ field ] : sorting_keys . append ( ( field , pad_field ) ) iterator = BucketIterator ( sorting_keys = sorting_keys , max_instances_in_memory = 10000 , batch_size = batch_size , biggest_batch_first = True , ) tr_generator = iterator ( task . train_data , num_epochs = None ) tr_generator = move_to_device ( tr_generator , self . _cuda_device ) task_info [ "iterator" ] = iterator if phase == "pretrain" : log . info ( f'Task {task.name} has pretrain fraction {task.pretrain_fraction}' ) task_info [ "n_tr_batches" ] = math . ceil ( task . n_train_examples * task . pretrain_fraction / batch_size ) log . info ( f'Task {task.name} will have {math.ceil(task.n_train_examples * task.pretrain_fraction)} many training examples after subsampling.' ) else : task_info [ "n_tr_batches" ] = math . ceil ( task . n_train_examples / batch_size ) task_info [ "tr_generator" ] = tr_generator task_info [ "loss" ] = 0.0 task_info [ "total_batches_trained" ] = 0 task_info [ "n_batches_since_val" ] = 0 opt_params = copy . deepcopy ( optimizer_params ) if self . _max_epochs > 0 and "t_total" in optimizer_params : opt_params [ "t_total" ] = task_info [ "n_tr_batches" ] * self . _max_epochs task_info [ "optimizer" ] = Optimizer . from_params ( train_params , opt_params ) task_info [ "scheduler" ] = LearningRateScheduler . from_params ( task_info [ "optimizer" ] , copy . deepcopy ( scheduler_params ) ) task_info [ "stopped" ] = False task_info [ "last_log" ] = time . time ( ) all_metrics = [ task . val_metric for task in tasks ] + [ "micro_avg" , "macro_avg" ] metric_infos = { metric : { "hist" : [ ] , "stopped" : False , "best" : ( - 1 , { } ) } for metric in all_metrics } self . task_to_metric_mapping = { task . val_metric : task . name for task in tasks } self . _task_infos = task_infos self . _metric_infos = metric_infos return task_infos , metric_infos
Set up the trainer by initializing task_infos and metric_infos, which
track necessary information about the training status of each task and metric respectively.
Set up the trainer by initializing task_infos and metric_infos which track necessary information about the training status of each task and metric respectively .
""" Set up the trainer by initializing task_infos and metric_infos, which
        track necessary information about the training status of each task and metric respectively.

        Returns:
            - task_infos (Dict[str:Dict[str:???]]): dictionary containing where each task_info
              contains:
                - iterator: a task specific (because it uses that task's fields to dynamically
                    batch) batcher
                - n_tr_batches: the number of training batches
                - tr_generator: generator object that returns the batches, set to repeat
                    indefinitely
                - loss: the accumulated loss (during training or validation)
                - n_batches_since_val: number of batches trained on since the last validation
                - total_batches_trained: number of batches trained over all validation checks
                - optimizer: a task specific optimizer, not used if the global optimizer is not
                    None
                - scheduler: a task specific scheduler, not used if the global optimizer is not
                    None
                - stopped: a bool indicating if that task is stopped or not (if it ran out of
                    patience or hit min lr)
                - last_log: the time we last logged progress for the task

            - metric_infos (Dict[str:Dict[str:???]]): dictionary containing metric information.
                Each metric should be the validation metric of a task, except {micro/macro}_avg,
                which are privileged to get an aggregate multi-task score. Each dict contains:
                - hist (List[float]): previous values of the metric
                - stopped (Bool): whether or not that metric is stopped or not
                - best (Tuple(Int, Dict)): information on the best value of that metric and when
                    it happened
        """ # Adding task-specific smart iterator to speed up training # Warning: This won't be precise when training_data_fraction is set, since each #  example is included or excluded independently using a hashing function. # Fortunately, it doesn't need to be. #task.n_train_examples * self._training_data_fraction / batch_size # Original #XXX # deepcopy b/c using Params pops values and we may want to reuse # the Params object later # If we know in advance how many opt steps for the transformer # there are, set it. # Metric bookkeeping
task_infos (Dict[str:Dict[str:???]]): dictionary containing where each task_info
contains:
iterator: a task specific (because it uses that task's fields to dynamically
batch) batcher
n_tr_batches: the number of training batches
tr_generator: generator object that returns the batches, set to repeat
indefinitely
loss: the accumulated loss (during training or validation)
n_batches_since_val: number of batches trained on since the last validation
total_batches_trained: number of batches trained over all validation checks
optimizer: a task specific optimizer, not used if the global optimizer is not
None
scheduler: a task specific scheduler, not used if the global optimizer is not
None
stopped: a bool indicating if that task is stopped or not (if it ran out of
patience or hit min lr)
last_log: the time we last logged progress for the task

metric_infos (Dict[str:Dict[str:???]]): dictionary containing metric information.
Each metric should be the validation metric of a task, except {micro/macro}_avg,
which are privileged to get an aggregate multi-task score. Each dict contains:
hist (List[float]): previous values of the metric
stopped (Bool): whether or not that metric is stopped or not
best (Tuple(Int, Dict)): information on the best value of that metric and when
it happened
task_infos ( Dict [ str : Dict [ str : ??? ]] ) : dictionary containing where each task_info contains : iterator : a task specific ( because it uses that task ' s fields to dynamically batch ) batcher n_tr_batches : the number of training batches tr_generator : generator object that returns the batches set to repeat indefinitely loss : the accumulated loss ( during training or validation ) n_batches_since_val : number of batches trained on since the last validation total_batches_trained : number of batches trained over all validation checks optimizer : a task specific optimizer not used if the global optimizer is not None scheduler : a task specific scheduler not used if the global optimizer is not None stopped : a bool indicating if that task is stopped or not ( if it ran out of patience or hit min lr ) last_log : the time we last logged progress for the task metric_infos ( Dict [ str : Dict [ str : ??? ]] ) : dictionary containing metric information . Each metric should be the validation metric of a task except { micro / macro } _avg which are privileged to get an aggregate multi - task score . Each dict contains : hist ( List [ float ] ) : previous values of the metric stopped ( Bool ) : whether or not that metric is stopped or not best ( Tuple ( Int Dict )) : information on the best value of that metric and when it happened
self
tasks
batch_size
train_params
optimizer_params
scheduler_params
phase