





'''simple docstring'''




import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowercase__       : str         =       16
lowercase__       : Optional[int]         =       32






def 			_lowerCAmelCase     (    __snake_case    :					Accelerator					,				__snake_case    :					int = 16     )	->	Any:
     __A		:					Optional[Any]						=			AutoTokenizer.from_pretrained('bert-base-cased'     )
     __A		:					str						=			load_dataset('glue'					,				'mrpc'     )

     def tokenize_function(__snake_case    :					str     ):
          # max_length=None => use the model max length (it's actually the default)
          __A		:					Any						=			tokenizer(examples['sentence1']					,				examples['sentence2']					,				truncation=__snake_case					,				max_length=__snake_case     )
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     # starting with the main process first:
     with accelerator.main_process_first():
          __A		:					int						=			datasets.map(
              __snake_case					,				batched=__snake_case					,				remove_columns=['idx', 'sentence1', 'sentence2']					,				)

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     __A		:					Optional[int]						=			tokenized_datasets.rename_column('label'					,				'labels'     )

     def collate_fn(__snake_case    :					Dict     ):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          __A		:					Optional[int]						=			1_28 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
               __A		:					int						=			16
          elif accelerator.mixed_precision != "no":
               __A		:					Optional[int]						=			8
          else:
               __A		:					Any						=			None

          return tokenizer.pad(
              __snake_case					,				padding='longest'					,				max_length=__snake_case					,				pad_to_multiple_of=__snake_case					,				return_tensors='pt'					,				)

     # Instantiate dataloaders.
     __A		:					str						=			DataLoader(
         tokenized_datasets['train']					,				shuffle=__snake_case					,				collate_fn=__snake_case					,				batch_size=__snake_case     )
     __A		:					Optional[Any]						=			DataLoader(
         tokenized_datasets['validation']					,				shuffle=__snake_case					,				collate_fn=__snake_case					,				batch_size=__snake_case     )

     return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  lowercase__       : Tuple         =       mocked_dataloaders  # noqa: F811






def 			_lowerCAmelCase     (    __snake_case    :					Optional[int]					,				__snake_case    :					int     )	->	Optional[Any]:
     # For testing only
     if os.environ.get('TESTING_MOCKED_DATALOADERS'					,				__snake_case     ) == "1":
          __A		:					List[str]						=			2
     # Initialize accelerator
     __A		:					Optional[int]						=			Accelerator(cpu=args.cpu					,				mixed_precision=args.mixed_precision     )
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     __A		:					Any						=			config['lr']
     __A		:					Any						=			int(config['num_epochs']     )
     __A		:					List[str]						=			int(config['seed']     )
     __A		:					List[str]						=			int(config['batch_size']     )

     __A		:					Dict						=			evaluate.load('glue'					,				'mrpc'     )

     # New Code #
     # We now can define an inner training loop function. It should take a batch size as the only parameter,
     # and build the dataloaders in there.
     # It also gets our decorator
     @find_executable_batch_size(starting_batch_size=__snake_case     )
     def inner_training_loop(__snake_case    :					int     ):
          # And now just move everything below under this function
          # We need to bring in the Accelerator object from earlier
          nonlocal accelerator
          # And reset all of its attributes that could hold onto any memory:
          accelerator.free_memory()

          # Then we can declare the model, optimizer, and everything else:
          set_seed(__snake_case     )

          # Instantiate the model (we build the model here so that the seed also control new weights initialization)
          __A		:					Optional[Any]						=			AutoModelForSequenceClassification.from_pretrained('bert-base-cased'					,				return_dict=__snake_case     )

          # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
          # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
          # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
          __A		:					Optional[Any]						=			model.to(accelerator.device     )

          # Instantiate optimizer
          __A		:					Optional[int]						=			AdamW(params=model.parameters()					,				lr=__snake_case     )
          __A					,__A		:					Tuple						=			get_dataloaders(__snake_case					,				__snake_case     )

          # Instantiate scheduler
          __A		:					Tuple						=			get_linear_schedule_with_warmup(
              optimizer=__snake_case					,				num_warmup_steps=1_00					,				num_training_steps=(len(__snake_case     ) * num_epochs)					,				)

          # Prepare everything
          # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
          # prepare method.
          __A					,__A					,__A					,__A					,__A		:					List[Any]						=			accelerator.prepare(
              __snake_case					,				__snake_case					,				__snake_case					,				__snake_case					,				__snake_case     )

          # Now we train the model
          for epoch in range(__snake_case     ):
               model.train()
               for step, batch in enumerate(__snake_case     ):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device     )
                    __A		:					str						=			model(**__snake_case     )
                    __A		:					Union[str, Any]						=			outputs.loss
                    accelerator.backward(__snake_case     )
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

               model.eval()
               for step, batch in enumerate(__snake_case     ):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device     )
                    with torch.no_grad():
                         __A		:					Tuple						=			model(**__snake_case     )
                    __A		:					List[Any]						=			outputs.logits.argmax(dim=-1     )
                    __A					,__A		:					Any						=			accelerator.gather_for_metrics((predictions, batch['labels'])     )
                    metric.add_batch(
                        predictions=__snake_case					,				references=__snake_case					,				)

               __A		:					int						=			metric.compute()
               # Use accelerator.print to print only on the main process.
               accelerator.print(f'epoch {epoch}:'					,				__snake_case     )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
     inner_training_loop()






def 			_lowerCAmelCase     (    )	->	Tuple:
     __A		:					Tuple						=			argparse.ArgumentParser(description='Simple example of training script.'     )
     parser.add_argument(
         '--mixed_precision'					,				type=__snake_case					,				default=__snake_case					,				choices=['no', 'fp16', 'bf16', 'fp8']					,				help='Whether to use mixed precision. Choose'
         'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
         'and an Nvidia Ampere GPU.'					,				)
     parser.add_argument('--cpu'					,				action='store_true'					,				help='If passed, will train on the CPU.'     )
     __A		:					Optional[Any]						=			parser.parse_args()
     __A		:					Union[str, Any]						=			{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
     training_function(__snake_case					,				__snake_case     )


if __name__ == "__main__":
  main()





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A								=       logging.get_logger(__name__)

__A								=       {
    "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json",
}







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			'''lxmert'''
			snake_case_  						=			{}


			def __init__( self     ,		lowerCamelCase__=30_522     ,		lowerCamelCase__=768     ,		lowerCamelCase__=12     ,		lowerCamelCase__=9_500     ,		lowerCamelCase__=1_600     ,		lowerCamelCase__=400     ,		lowerCamelCase__=3_072     ,		lowerCamelCase__="gelu"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=512     ,		lowerCamelCase__=2     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=1e-12     ,		lowerCamelCase__=9     ,		lowerCamelCase__=5     ,		lowerCamelCase__=5     ,		lowerCamelCase__=2_048     ,		lowerCamelCase__=4     ,		lowerCamelCase__=6.67     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		lowerCamelCase__=True     ,		**lowerCamelCase__     ,		)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       hidden_size
							__lowerCamelCase									=       num_attention_heads
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       intermediate_size
							__lowerCamelCase									=       hidden_dropout_prob
							__lowerCamelCase									=       attention_probs_dropout_prob
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       initializer_range
							__lowerCamelCase									=       layer_norm_eps
							__lowerCamelCase									=       num_qa_labels
							__lowerCamelCase									=       num_object_labels
							__lowerCamelCase									=       num_attr_labels
							__lowerCamelCase									=       l_layers
							__lowerCamelCase									=       x_layers
							__lowerCamelCase									=       r_layers
							__lowerCamelCase									=       visual_feat_dim
							__lowerCamelCase									=       visual_pos_dim
							__lowerCamelCase									=       visual_loss_normalizer
							__lowerCamelCase									=       task_matched
							__lowerCamelCase									=       task_mask_lm
							__lowerCamelCase									=       task_obj_predict
							__lowerCamelCase									=       task_qa
							__lowerCamelCase									=       visual_obj_loss
							__lowerCamelCase									=       visual_attr_loss
							__lowerCamelCase									=       visual_feat_loss
							__lowerCamelCase									=       {'vision': r_layers, 'cross_encoder': x_layers, 'language': l_layers}
							super().__init__(**lowerCamelCase__				)


