



'''simple docstring'''





import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowercase_	 =						16
lowercase_	 =						32




def lowerCAmelCase						(__A						,			__A = 16):



     """simple docstring"""



     _a											=						AutoTokenizer.from_pretrained('''bert-base-cased''')
     _a											=						load_dataset('''glue'''						,			'''mrpc''')

     def tokenize_function(__A):
          # max_length=None => use the model max length (it's actually the default)
          _a											=						tokenizer(examples['''sentence1''']						,			examples['''sentence2''']						,			truncation=__A						,			max_length=__A)
          return outputs

     # Apply the method we just defined to all the examples in all the splits of the dataset
     # starting with the main process first:
     with accelerator.main_process_first():
          _a											=						datasets.map(
              __A						,			batched=__A						,			remove_columns=['''idx''', '''sentence1''', '''sentence2''']						,			)

     # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
     # transformers library
     _a											=						tokenized_datasets.rename_column('''label'''						,			'''labels''')

     def collate_fn(__A):
          # On TPU it's best to pad everything to the same length or training will be very slow.
          _a											=						128 if accelerator.distributed_type == DistributedType.TPU else None
          # When using mixed precision we want round multiples of 8/16
          if accelerator.mixed_precision == "fp8":
               _a											=						16
          elif accelerator.mixed_precision != "no":
               _a											=						8
          else:
               _a											=						None

          return tokenizer.pad(
              __A						,			padding='''longest'''						,			max_length=__A						,			pad_to_multiple_of=__A						,			return_tensors='''pt'''						,			)

     # Instantiate dataloaders.
     _a											=						DataLoader(
         tokenized_datasets['''train''']						,			shuffle=__A						,			collate_fn=__A						,			batch_size=__A)
     _a											=						DataLoader(
         tokenized_datasets['''validation''']						,			shuffle=__A						,			collate_fn=__A						,			batch_size=__A)

     return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
  from accelerate.test_utils.training import mocked_dataloaders

  lowercase_	 =						mocked_dataloaders  # noqa: F811




def lowerCAmelCase						(__A						,			__A):



     """simple docstring"""



     if os.environ.get('''TESTING_MOCKED_DATALOADERS'''						,			__A) == "1":
          _a											=						2
     # Initialize accelerator
     _a											=						Accelerator(cpu=args.cpu						,			mixed_precision=args.mixed_precision)
     # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
     _a											=						config['''lr''']
     _a											=						int(config['''num_epochs'''])
     _a											=						int(config['''seed'''])
     _a											=						int(config['''batch_size'''])

     _a											=						evaluate.load('''glue'''						,			'''mrpc''')

     # New Code #
     # We now can define an inner training loop function. It should take a batch size as the only parameter,
     # and build the dataloaders in there.
     # It also gets our decorator
     @find_executable_batch_size(starting_batch_size=__A)
     def inner_training_loop(__A):
          # And now just move everything below under this function
          # We need to bring in the Accelerator object from earlier
          nonlocal accelerator
          # And reset all of its attributes that could hold onto any memory:
          accelerator.free_memory()

          # Then we can declare the model, optimizer, and everything else:
          set_seed(__A)

          # Instantiate the model (we build the model here so that the seed also control new weights initialization)
          _a											=						AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''						,			return_dict=__A)

          # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
          # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
          # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
          _a											=						model.to(accelerator.device)

          # Instantiate optimizer
          _a											=						AdamW(params=model.parameters()						,			lr=__A)
          _a						,					_a											=						get_dataloaders(__A						,			__A)

          # Instantiate scheduler
          _a											=						get_linear_schedule_with_warmup(
              optimizer=__A						,			num_warmup_steps=100						,			num_training_steps=(len(__A) * num_epochs)						,			)

          # Prepare everything
          # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
          # prepare method.
          _a						,					_a						,					_a						,					_a						,					_a											=						accelerator.prepare(
              __A						,			__A						,			__A						,			__A						,			__A)

          # Now we train the model
          for epoch in range(__A):
               model.train()
               for step, batch in enumerate(__A):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device)
                    _a											=						model(**__A)
                    _a											=						outputs.loss
                    accelerator.backward(__A)
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()

               model.eval()
               for step, batch in enumerate(__A):
                    # We could avoid this line since we set the accelerator with `device_placement=True`.
                    batch.to(accelerator.device)
                    with torch.no_grad():
                         _a											=						model(**__A)
                    _a											=						outputs.logits.argmax(dim=-1)
                    _a						,					_a											=						accelerator.gather_for_metrics((predictions, batch['''labels''']))
                    metric.add_batch(
                        predictions=__A						,			references=__A						,			)

               _a											=						metric.compute()
               # Use accelerator.print to print only on the main process.
               accelerator.print(F'''epoch {epoch}:'''						,			__A)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
     inner_training_loop()




def lowerCAmelCase						():



     """simple docstring"""



     _a											=						argparse.ArgumentParser(description='''Simple example of training script.''')
     parser.add_argument(
         '''--mixed_precision'''						,			type=__A						,			default=__A						,			choices=['''no''', '''fp16''', '''bf16''', '''fp8''']						,			help='''Whether to use mixed precision. Choose'''
         '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
         '''and an Nvidia Ampere GPU.'''						,			)
     parser.add_argument('''--cpu'''						,			action='''store_true'''						,			help='''If passed, will train on the CPU.''')
     _a											=						parser.parse_args()
     _a											=						{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
     training_function(__A						,			__A)


if __name__ == "__main__":
  main()








'''simple docstring'''





import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoImageProcessor, ViTImageProcessor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


lowercase_	 =						get_tests_dir("fixtures")






class __A    (						unittest.TestCase		):



 '''simple docstring'''







 def       a__						(self )					->  List[Any]:


      """simple docstring"""



      _a											=						mock.Mock()
      _a											=						500
      _a											=						{}
      _a											=						HTTPError
      _a											=						{}

      # Download this model to make sure it's in the cache.
      _a											=						ViTImageProcessor.from_pretrained('''hf-internal-testing/tiny-random-vit''' )
      # Under the mock environment we get a 500 error when trying to reach the model.
      with mock.patch('''requests.Session.request'''				,     return_value=A ) as mock_head:
           _a											=						ViTImageProcessor.from_pretrained('''hf-internal-testing/tiny-random-vit''' )
           # This check we did call the fake head request
           mock_head.assert_called()



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						ViTImageProcessor.from_pretrained(
          '''https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json''' )



 def       a__						(self )					->  Dict:


      """simple docstring"""



      with self.assertRaises(A ):
           # config is in subfolder, the following should not work without specifying the subfolder
           _a											=						AutoImageProcessor.from_pretrained('''hf-internal-testing/stable-diffusion-all-variants''' )

      _a											=						AutoImageProcessor.from_pretrained(
          '''hf-internal-testing/stable-diffusion-all-variants'''				,     subfolder='''feature_extractor''' )

      self.assertIsNotNone(A )






@is_staging_test
class __A    (						unittest.TestCase		):



 '''simple docstring'''







 @classmethod
 def       a__						(cls )					->  Any:


      """simple docstring"""



      _a											=						TOKEN
      HfFolder.save_token(A )



 @classmethod
 def       a__						(cls )					->  Union[str, Any]:


      """simple docstring"""



      try:
           delete_repo(token=cls._token				,     repo_id='''test-image-processor''' )
      except HTTPError:
           pass

      try:
           delete_repo(token=cls._token				,     repo_id='''valid_org/test-image-processor-org''' )
      except HTTPError:
           pass

      try:
           delete_repo(token=cls._token				,     repo_id='''test-dynamic-image-processor''' )
      except HTTPError:
           pass



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						ViTImageProcessor.from_pretrained(A )
      image_processor.push_to_hub('''test-image-processor'''				,     use_auth_token=self._token )

      _a											=						ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor''' )
      for k, v in image_processor.__dict__.items():
           self.assertEqual(A				,     getattr(A				,     A ) )

      # Reset repo
      delete_repo(token=self._token				,     repo_id='''test-image-processor''' )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
           image_processor.save_pretrained(
               A				,     repo_id='''test-image-processor'''				,     push_to_hub=A				,     use_auth_token=self._token )

      _a											=						ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor''' )
      for k, v in image_processor.__dict__.items():
           self.assertEqual(A				,     getattr(A				,     A ) )



 def       a__						(self )					->  List[Any]:


      """simple docstring"""



      _a											=						ViTImageProcessor.from_pretrained(A )
      image_processor.push_to_hub('''valid_org/test-image-processor'''				,     use_auth_token=self._token )

      _a											=						ViTImageProcessor.from_pretrained('''valid_org/test-image-processor''' )
      for k, v in image_processor.__dict__.items():
           self.assertEqual(A				,     getattr(A				,     A ) )

      # Reset repo
      delete_repo(token=self._token				,     repo_id='''valid_org/test-image-processor''' )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
           image_processor.save_pretrained(
               A				,     repo_id='''valid_org/test-image-processor-org'''				,     push_to_hub=A				,     use_auth_token=self._token )

      _a											=						ViTImageProcessor.from_pretrained('''valid_org/test-image-processor-org''' )
      for k, v in image_processor.__dict__.items():
           self.assertEqual(A				,     getattr(A				,     A ) )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      CustomImageProcessor.register_for_auto_class()
      _a											=						CustomImageProcessor.from_pretrained(A )

      image_processor.push_to_hub('''test-dynamic-image-processor'''				,     use_auth_token=self._token )

      # This has added the proper auto_map field to the config
      self.assertDictEqual(
          image_processor.auto_map				,     {'''AutoImageProcessor''': '''custom_image_processing.CustomImageProcessor'''}				,     )

      _a											=						AutoImageProcessor.from_pretrained(
          f'''{USER}/test-dynamic-image-processor'''				,     trust_remote_code=A )
      # Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module
      self.assertEqual(new_image_processor.__class__.__name__				,     '''CustomImageProcessor''' )



