def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id, lang_adapter_names, task_name, lang2id=None):
    'Train the model.'
    if (args.local_rank in [(- 1), 0]):
        tb_writer = SummaryWriter()
    args.train_batch_size = (args.per_gpu_train_batch_size * max(1, args.n_gpu))
    print(f'Local Rank = {args.local_rank}')
    print(len(train_dataset))
    train_sampler = (RandomSampler(train_dataset) if (args.local_rank == (- 1)) else DistributedSampler(train_dataset))
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
    if (args.max_steps > 0):
        t_total = args.max_steps
        args.num_train_epochs = ((args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps)) + 1)
    else:
        t_total = ((len(train_dataloader) // args.gradient_accumulation_steps) * args.num_train_epochs)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [{'params': [p for (n, p) in model.named_parameters() if (not any(((nd in n) for nd in no_decay)))], 'weight_decay': args.weight_decay}, {'params': [p for (n, p) in model.named_parameters() if any(((nd in n) for nd in no_decay))], 'weight_decay': 0.0}]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    logging.info([n for (n, p) in model.named_parameters() if p.requires_grad])
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
    if args.fp16:
        try:
            from apex import amp
        except ImportError:
            raise ImportError('Please install apex from https://www.github.com/nvidia/apex to use fp16 training.')
        (model, optimizer) = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    if (args.local_rank != (- 1)):
        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True)
    logger.info('***** Running training *****')
    logger.info('  Num examples = %d', len(train_dataset))
    logger.info('  Num Epochs = %d', args.num_train_epochs)
    logger.info('  Instantaneous batch size per GPU = %d', args.per_gpu_train_batch_size)
    logger.info('  Total train batch size (w. parallel, distributed & accumulation) = %d', ((args.train_batch_size * args.gradient_accumulation_steps) * (torch.distributed.get_world_size() if (args.local_rank != (- 1)) else 1)))
    logger.info('  Gradient Accumulation steps = %d', args.gradient_accumulation_steps)
    logger.info('  Total optimization steps = %d', t_total)
    best_score = 0.0
    best_checkpoint = None
    patience = 0
    global_step = 0
    (tr_loss, logging_loss) = (0.0, 0.0)
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc='Epoch', disable=(args.local_rank not in [(- 1), 0]))
    set_seed(args)
    cur_epoch = 0
    for _ in train_iterator:
        epoch_iterator = tqdm(train_dataloader, desc='Iteration', disable=(args.local_rank not in [(- 1), 0]))
        cur_epoch += 1
        for (step, batch) in enumerate(epoch_iterator):
            batch = tuple((t.to(args.device) for t in batch if (t is not None)))
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            outputs = model(**inputs)
            loss = outputs[0]
            if (args.n_gpu > 1):
                loss = loss.mean()
            if (args.gradient_accumulation_steps > 1):
                loss = (loss / args.gradient_accumulation_steps)
            if args.fp16:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                loss.backward()
            tr_loss += loss.item()
            if (((step + 1) % args.gradient_accumulation_steps) == 0):
                if args.fp16:
                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
                else:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                scheduler.step()
                optimizer.step()
                model.zero_grad()
                global_step += 1
                if ((args.local_rank in [(- 1), 0]) and (args.logging_steps > 0) and ((global_step % args.logging_steps) == 0)):
                    if ((args.local_rank == (- 1)) and args.evaluate_during_training):
                        (results, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name)
                        for (key, value) in results.items():
                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
                    tb_writer.add_scalar('loss', ((tr_loss - logging_loss) / args.logging_steps), global_step)
                    logging_loss = tr_loss
                if ((args.local_rank in [(- 1), 0]) and (args.save_steps > 0) and ((global_step % args.save_steps) == 0)):
                    if args.save_only_best_checkpoint:
                        (result, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', prefix=global_step, lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name)
                        if (result['f1'] > best_score):
                            logger.info("result['f1']={} > best_score={}".format(result['f1'], best_score))
                            best_score = result['f1']
                            output_dir = os.path.join(args.output_dir, 'checkpoint-best')
                            best_checkpoint = output_dir
                            if (not os.path.exists(output_dir)):
                                os.makedirs(output_dir)
                            model_to_save = (model.module if hasattr(model, 'module') else model)
                            if args.do_save_adapters:
                                model_to_save.save_all_adapters(output_dir)
                            if args.do_save_adapter_fusions:
                                model_to_save.save_all_adapter_fusions(output_dir)
                            if args.do_save_full_model:
                                model_to_save.save_pretrained(output_dir)
                            torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                            logger.info('Saving the best model checkpoint to %s', output_dir)
                            logger.info('Reset patience to 0')
                            patience = 0
                        else:
                            patience += 1
                            logger.info('Hit patience={}'.format(patience))
                            if ((args.eval_patience > 0) and (patience > args.eval_patience)):
                                logger.info('early stop! patience={}'.format(patience))
                                epoch_iterator.close()
                                train_iterator.close()
                                if (args.local_rank in [(- 1), 0]):
                                    tb_writer.close()
                                return (global_step, (tr_loss / global_step))
                    else:
                        output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
                        if (not os.path.exists(output_dir)):
                            os.makedirs(output_dir)
                        model_to_save = (model.module if hasattr(model, 'module') else model)
                        if args.do_save_adapters:
                            model_to_save.save_all_adapters(output_dir)
                        if args.do_save_adapter_fusions:
                            model_to_save.save_all_adapter_fusions(output_dir)
                        if args.do_save_full_model:
                            model_to_save.save_pretrained(output_dir)
                        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                        logger.info('Saving model checkpoint to %s', output_dir)
            if ((args.max_steps > 0) and (global_step > args.max_steps)):
                epoch_iterator.close()
                break
        if ((args.max_steps > 0) and (global_step > args.max_steps)):
            train_iterator.close()
            break
    if (args.local_rank in [(- 1), 0]):
        tb_writer.close()
    return (global_step, (tr_loss / global_step))
Train the model.
def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id, lang_adapter_names, task_name, lang2id=None):
    
    if (args.local_rank in [(- 1), 0]):
        tb_writer = SummaryWriter()
    args.train_batch_size = (args.per_gpu_train_batch_size * max(1, args.n_gpu))
    print(f'Local Rank = {args.local_rank}')
    print(len(train_dataset))
    train_sampler = (RandomSampler(train_dataset) if (args.local_rank == (- 1)) else DistributedSampler(train_dataset))
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
    if (args.max_steps > 0):
        t_total = args.max_steps
        args.num_train_epochs = ((args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps)) + 1)
    else:
        t_total = ((len(train_dataloader) // args.gradient_accumulation_steps) * args.num_train_epochs)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [{'params': [p for (n, p) in model.named_parameters() if (not any(((nd in n) for nd in no_decay)))], 'weight_decay': args.weight_decay}, {'params': [p for (n, p) in model.named_parameters() if any(((nd in n) for nd in no_decay))], 'weight_decay': 0.0}]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    logging.info([n for (n, p) in model.named_parameters() if p.requires_grad])
    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
    if args.fp16:
        try:
            from apex import amp
        except ImportError:
            raise ImportError('Please install apex from https://www.github.com/nvidia/apex to use fp16 training.')
        (model, optimizer) = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
    if (args.n_gpu > 1):
        model = torch.nn.DataParallel(model)
    if (args.local_rank != (- 1)):
        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True)
    logger.info('***** Running training *****')
    logger.info('  Num examples = %d', len(train_dataset))
    logger.info('  Num Epochs = %d', args.num_train_epochs)
    logger.info('  Instantaneous batch size per GPU = %d', args.per_gpu_train_batch_size)
    logger.info('  Total train batch size (w. parallel, distributed & accumulation) = %d', ((args.train_batch_size * args.gradient_accumulation_steps) * (torch.distributed.get_world_size() if (args.local_rank != (- 1)) else 1)))
    logger.info('  Gradient Accumulation steps = %d', args.gradient_accumulation_steps)
    logger.info('  Total optimization steps = %d', t_total)
    best_score = 0.0
    best_checkpoint = None
    patience = 0
    global_step = 0
    (tr_loss, logging_loss) = (0.0, 0.0)
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc='Epoch', disable=(args.local_rank not in [(- 1), 0]))
    set_seed(args)
    cur_epoch = 0
    for _ in train_iterator:
        epoch_iterator = tqdm(train_dataloader, desc='Iteration', disable=(args.local_rank not in [(- 1), 0]))
        cur_epoch += 1
        for (step, batch) in enumerate(epoch_iterator):
            batch = tuple((t.to(args.device) for t in batch if (t is not None)))
            inputs = {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}
            if (args.model_type != 'distilbert'):
                inputs['token_type_ids'] = (batch[2] if (args.model_type in ['bert', 'xlnet']) else None)
            if (args.model_type == 'xlm'):
                inputs['langs'] = batch[4]
            outputs = model(**inputs)
            loss = outputs[0]
            if (args.n_gpu > 1):
                loss = loss.mean()
            if (args.gradient_accumulation_steps > 1):
                loss = (loss / args.gradient_accumulation_steps)
            if args.fp16:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                loss.backward()
            tr_loss += loss.item()
            if (((step + 1) % args.gradient_accumulation_steps) == 0):
                if args.fp16:
                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
                else:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                scheduler.step()
                optimizer.step()
                model.zero_grad()
                global_step += 1
                if ((args.local_rank in [(- 1), 0]) and (args.logging_steps > 0) and ((global_step % args.logging_steps) == 0)):
                    if ((args.local_rank == (- 1)) and args.evaluate_during_training):
                        (results, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name)
                        for (key, value) in results.items():
                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
                    tb_writer.add_scalar('loss', ((tr_loss - logging_loss) / args.logging_steps), global_step)
                    logging_loss = tr_loss
                if ((args.local_rank in [(- 1), 0]) and (args.save_steps > 0) and ((global_step % args.save_steps) == 0)):
                    if args.save_only_best_checkpoint:
                        (result, _) = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode='dev', prefix=global_step, lang=args.train_langs, lang2id=lang2id, lang_adapter_names=lang_adapter_names, task_name=task_name)
                        if (result['f1'] > best_score):
                            logger.info("result['f1']={} > best_score={}".format(result['f1'], best_score))
                            best_score = result['f1']
                            output_dir = os.path.join(args.output_dir, 'checkpoint-best')
                            best_checkpoint = output_dir
                            if (not os.path.exists(output_dir)):
                                os.makedirs(output_dir)
                            model_to_save = (model.module if hasattr(model, 'module') else model)
                            if args.do_save_adapters:
                                model_to_save.save_all_adapters(output_dir)
                            if args.do_save_adapter_fusions:
                                model_to_save.save_all_adapter_fusions(output_dir)
                            if args.do_save_full_model:
                                model_to_save.save_pretrained(output_dir)
                            torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                            logger.info('Saving the best model checkpoint to %s', output_dir)
                            logger.info('Reset patience to 0')
                            patience = 0
                        else:
                            patience += 1
                            logger.info('Hit patience={}'.format(patience))
                            if ((args.eval_patience > 0) and (patience > args.eval_patience)):
                                logger.info('early stop! patience={}'.format(patience))
                                epoch_iterator.close()
                                train_iterator.close()
                                if (args.local_rank in [(- 1), 0]):
                                    tb_writer.close()
                                return (global_step, (tr_loss / global_step))
                    else:
                        output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
                        if (not os.path.exists(output_dir)):
                            os.makedirs(output_dir)
                        model_to_save = (model.module if hasattr(model, 'module') else model)
                        if args.do_save_adapters:
                            model_to_save.save_all_adapters(output_dir)
                        if args.do_save_adapter_fusions:
                            model_to_save.save_all_adapter_fusions(output_dir)
                        if args.do_save_full_model:
                            model_to_save.save_pretrained(output_dir)
                        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                        logger.info('Saving model checkpoint to %s', output_dir)
            if ((args.max_steps > 0) and (global_step > args.max_steps)):
                epoch_iterator.close()
                break
        if ((args.max_steps > 0) and (global_step > args.max_steps)):
            train_iterator.close()
            break
    if (args.local_rank in [(- 1), 0]):
        tb_writer.close()
    return (global_step, (tr_loss / global_step))