





'''simple docstring'''

import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a   	=    16
a   	=    32

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase = 16   )    ->       List[Any]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							AutoTokenizer.from_pretrained("""bert-base-cased"""   )
   __SCREAMING_SNAKE_CASE   				=							load_dataset("""glue"""				,						"""mrpc"""   )

   def tokenize_function(__UpperCAmelCase   ):
      # max_length=None => use the model max length (it's actually the default)
      __SCREAMING_SNAKE_CASE   				=							tokenizer(examples["""sentence1"""]				,						examples["""sentence2"""]				,						truncation=lowerCAmelCase_				,						max_length=lowerCAmelCase_   )
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      __SCREAMING_SNAKE_CASE   				=							datasets.map(
          lowerCAmelCase_				,						batched=lowerCAmelCase_				,						remove_columns=["""idx""", """sentence1""", """sentence2"""]				,						)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   __SCREAMING_SNAKE_CASE   				=							tokenized_datasets.rename_column("""label"""				,						"""labels"""   )

   def collate_fn(__UpperCAmelCase   ):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      __SCREAMING_SNAKE_CASE   				=							128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         __SCREAMING_SNAKE_CASE   				=							16
      elif accelerator.mixed_precision != "no":
         __SCREAMING_SNAKE_CASE   				=							8
      else:
         __SCREAMING_SNAKE_CASE   				=							None

      return tokenizer.pad(
          lowerCAmelCase_				,						padding="""longest"""				,						max_length=lowerCAmelCase_				,						pad_to_multiple_of=lowerCAmelCase_				,						return_tensors="""pt"""				,						)

   # Instantiate dataloaders.
   __SCREAMING_SNAKE_CASE   				=							DataLoader(
       tokenized_datasets["""train"""]				,						shuffle=lowerCAmelCase_				,						collate_fn=lowerCAmelCase_				,						batch_size=lowerCAmelCase_   )
   __SCREAMING_SNAKE_CASE   				=							DataLoader(
       tokenized_datasets["""validation"""]				,						shuffle=lowerCAmelCase_				,						collate_fn=lowerCAmelCase_				,						batch_size=lowerCAmelCase_   )

   return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
    from accelerate.test_utils.training import mocked_dataloaders

    a   	=    mocked_dataloaders  # noqa: F811

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       Optional[Any]:

   '''simple docstring'''







   if os.environ.get("""TESTING_MOCKED_DATALOADERS"""				,						lowerCAmelCase_   ) == "1":
      __SCREAMING_SNAKE_CASE   				=							2
   # Initialize accelerator
   __SCREAMING_SNAKE_CASE   				=							Accelerator(cpu=args.cpu				,						mixed_precision=args.mixed_precision   )
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   __SCREAMING_SNAKE_CASE   				=							config["""lr"""]
   __SCREAMING_SNAKE_CASE   				=							int(config["""num_epochs"""]   )
   __SCREAMING_SNAKE_CASE   				=							int(config["""seed"""]   )
   __SCREAMING_SNAKE_CASE   				=							int(config["""batch_size"""]   )

   __SCREAMING_SNAKE_CASE   				=							evaluate.load("""glue"""				,						"""mrpc"""   )

   # If the batch size is too big we use gradient accumulation
   __SCREAMING_SNAKE_CASE   				=							1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      __SCREAMING_SNAKE_CASE   				=							batch_size // MAX_GPU_BATCH_SIZE
      __SCREAMING_SNAKE_CASE   				=							MAX_GPU_BATCH_SIZE

   set_seed(lowerCAmelCase_   )
   __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							get_dataloaders(lowerCAmelCase_				,						lowerCAmelCase_   )
   # Instantiate the model (we build the model here so that the seed also control new weights initialization)
   __SCREAMING_SNAKE_CASE   				=							AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""				,						return_dict=lowerCAmelCase_   )

   # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
   # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
   # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
   __SCREAMING_SNAKE_CASE   				=							model.to(accelerator.device   )

   # Instantiate optimizer
   __SCREAMING_SNAKE_CASE   				=							AdamW(params=model.parameters()				,						lr=lowerCAmelCase_   )

   # Instantiate scheduler
   __SCREAMING_SNAKE_CASE   				=							get_linear_schedule_with_warmup(
       optimizer=lowerCAmelCase_				,						num_warmup_steps=100				,						num_training_steps=(len(lowerCAmelCase_   ) * num_epochs) // gradient_accumulation_steps				,						)

   # Prepare everything
   # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
   # prepare method.
   __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							accelerator.prepare(
       lowerCAmelCase_				,						lowerCAmelCase_				,						lowerCAmelCase_				,						lowerCAmelCase_				,						lowerCAmelCase_   )

   # Now we train the model
   for epoch in range(lowerCAmelCase_   ):
      model.train()
      for step, batch in enumerate(lowerCAmelCase_   ):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device   )
         __SCREAMING_SNAKE_CASE   				=							model(**lowerCAmelCase_   )
         __SCREAMING_SNAKE_CASE   				=							outputs.loss
         __SCREAMING_SNAKE_CASE   				=							loss / gradient_accumulation_steps
         accelerator.backward(lowerCAmelCase_   )
         if step % gradient_accumulation_steps == 0:
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()

      model.eval()
      __SCREAMING_SNAKE_CASE   				=							0
      for step, batch in enumerate(lowerCAmelCase_   ):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device   )
         with torch.no_grad():
            __SCREAMING_SNAKE_CASE   				=							model(**lowerCAmelCase_   )
         __SCREAMING_SNAKE_CASE   				=							outputs.logits.argmax(dim=-1   )
         __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							accelerator.gather((predictions, batch["""labels"""])   )
         # New Code #
         # First we check if it's a distributed system
         if accelerator.use_distributed:
            # Then see if we're on the last batch of our eval dataloader
            if step == len(lowerCAmelCase_   ) - 1:
               # Last batch needs to be truncated on distributed systems as it contains additional samples
               __SCREAMING_SNAKE_CASE   				=							predictions[: len(eval_dataloader.dataset   ) - samples_seen]
               __SCREAMING_SNAKE_CASE   				=							references[: len(eval_dataloader.dataset   ) - samples_seen]
            else:
               # Otherwise we add the number of samples seen
               samples_seen += references.shape[0]
            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
            # accelerator.gather_for_metrics((predictions, batch["labels"]))
         metric.add_batch(
             predictions=lowerCAmelCase_				,						references=lowerCAmelCase_				,						)

      __SCREAMING_SNAKE_CASE   				=							metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(f"""epoch {epoch}:"""				,						lowerCAmelCase_   )

def     __magic_name__							(				)    ->       Optional[Any]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							argparse.ArgumentParser(description="""Simple example of training script."""   )
   parser.add_argument(
       """--mixed_precision"""				,						type=lowerCAmelCase_				,						default=lowerCAmelCase_				,						choices=["""no""", """fp16""", """bf16""", """fp8"""]				,						help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""				,						)
   parser.add_argument("""--cpu"""				,						action="""store_true"""				,						help="""If passed, will train on the CPU."""   )
   __SCREAMING_SNAKE_CASE   				=							parser.parse_args()
   __SCREAMING_SNAKE_CASE   				=							{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
   training_function(lowerCAmelCase_				,						lowerCAmelCase_   )


if __name__ == "__main__":
    main()












import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_A					:				Dict				=			get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")



class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Any				=	BartphoTokenizer
    lowerCamelCase__    :		Tuple				=	False
    lowerCamelCase__    :		Dict				=	True



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        SCREAMING_SNAKE_CASE__											=			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
        SCREAMING_SNAKE_CASE__											=			dict(zip(A_   ,				range(len(A_     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			{'''unk_token''': '''<unk>'''}

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''monolingual_vocab_file''']     )
        with open(self.monolingual_vocab_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as fp:
            for token in vocab_tokens:
                fp.write(f'''{token} {vocab_tokens[token]}\n'''     )

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        tokenizer.save_pretrained(self.tmpdirname     )



    def  lowercase_					(       self   ,				**A_     ):





        '''simple docstring'''

        kwargs.update(self.special_tokens_map     )
        return BartphoTokenizer.from_pretrained(self.tmpdirname   ,				**A_     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''This is a<unk><unk> test'''
        return input_text, output_text




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''▁This ▁is ▁a ▁l à ▁t est'''.split()
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE__											=			[4, 5, 6, 3, 3, 7, 8, 3]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A_     )   ,				A_     )
