





import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTJConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, tooslow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
			import jax
			import jax.numpy as jnp

			from transformers.modeling_flax_pytorch_utils import (
			    convert_pytorch_state_dict_to_flax,
			    load_flax_weights_in_pytorch_model,
			)
			from transformers.models.gptj.modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel

if is_torch_available():
			import torch
class   a_ :

			def __init__(   self   :  Optional[int]     , UpperCAmelCase__   :  int     , UpperCAmelCase__   :  Any=14     , UpperCAmelCase__   :  Any=7     , UpperCAmelCase__   :  Dict=True     , UpperCAmelCase__   :  Dict=True     , UpperCAmelCase__   :  Dict=False     , UpperCAmelCase__   :  Union[str, Any]=True     , UpperCAmelCase__   :  Optional[int]=99     , UpperCAmelCase__   :  Union[str, Any]=32     , UpperCAmelCase__   :  int=4     , UpperCAmelCase__   :  Any=4     , UpperCAmelCase__   :  str=4     , UpperCAmelCase__   :  Any=37     , UpperCAmelCase__   :  List[str]="gelu"     , UpperCAmelCase__   :  Union[str, Any]=0.1     , UpperCAmelCase__   :  List[Any]=0.1     , UpperCAmelCase__   :  Union[str, Any]=512     , UpperCAmelCase__   :  Dict=0.02     , ):






							"""simple docstring"""
							snake_case   :					Union[str, Any]    						=				parent
							snake_case   :					List[str]    						=				batch_size
							snake_case   :					Any    						=				seq_length
							snake_case   :					int    						=				is_training
							snake_case   :					Union[str, Any]    						=				use_input_mask
							snake_case   :					int    						=				use_token_type_ids
							snake_case   :					Any    						=				use_labels
							snake_case   :					Tuple    						=				vocab_size
							snake_case   :					Tuple    						=				hidden_size
							snake_case   :					Union[str, Any]    						=				rotary_dim
							snake_case   :					List[str]    						=				num_hidden_layers
							snake_case   :					Optional[Any]    						=				num_attention_heads
							snake_case   :					Tuple    						=				intermediate_size
							snake_case   :					Optional[int]    						=				hidden_act
							snake_case   :					List[str]    						=				hidden_dropout_prob
							snake_case   :					Dict    						=				attention_probs_dropout_prob
							snake_case   :					Union[str, Any]    						=				max_position_embeddings
							snake_case   :					List[str]    						=				initializer_range
							snake_case   :					Union[str, Any]    						=				None
							snake_case   :					Optional[Any]    						=				vocab_size - 1
							snake_case   :					str    						=				vocab_size - 1
							snake_case   :					Union[str, Any]    						=				vocab_size - 1

			def      lowerCAmelCase(   self   :  Dict						):






							"""simple docstring"""
							snake_case   :					List[Any]    						=				ids_tensor([self.batch_size, self.seq_length]     , self.vocab_size						)

							snake_case   :					Union[str, Any]    						=				None
							if self.use_input_mask:
											snake_case   :					str    						=				random_attention_mask([self.batch_size, self.seq_length]						)

							snake_case   :					str    						=				GPTJConfig(
							    vocab_size=self.vocab_size     , n_embd=self.hidden_size     , n_layer=self.num_hidden_layers     , n_head=self.num_attention_heads     , n_positions=self.max_position_embeddings     , use_cache=UpperCAmelCase__     , bos_token_id=self.bos_token_id     , eos_token_id=self.eos_token_id     , pad_token_id=self.pad_token_id     , rotary_dim=self.rotary_dim     , )

							return (config, input_ids, input_mask)

			def      lowerCAmelCase(   self   :  str						):






							"""simple docstring"""
							snake_case   :					Union[str, Any]    						=				self.prepare_config_and_inputs()
							snake_case						,						snake_case						,						snake_case   :					Tuple    						=				config_and_inputs
							snake_case   :					str    						=				{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
							return config, inputs_dict

			def      lowerCAmelCase(   self   :  Any     , UpperCAmelCase__   :  Optional[int]     , UpperCAmelCase__   :  Dict     , UpperCAmelCase__   :  List[str]     , UpperCAmelCase__   :  str						):






							"""simple docstring"""
							snake_case   :					List[str]    						=				20
							snake_case   :					int    						=				model_class_name(UpperCAmelCase__						)

							snake_case   :					int    						=				model.init_cache(input_ids.shape[0]     , UpperCAmelCase__						)
							snake_case   :					List[Any]    						=				jnp.ones((input_ids.shape[0], max_decoder_length)     , dtype='''i4'''						)

							snake_case   :					Optional[Any]    						=				jnp.broadcast_to(
							    jnp.arange(input_ids.shape[-1] - 1						)[None, :]     , (input_ids.shape[0], input_ids.shape[-1] - 1)						)
							snake_case   :					Union[str, Any]    						=				model(
							    input_ids[:, :-1]     , attention_mask=UpperCAmelCase__     , past_key_values=UpperCAmelCase__     , position_ids=UpperCAmelCase__     , )

							snake_case   :					Optional[Any]    						=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     , dtype='''i4'''						)
							snake_case   :					Dict    						=				model(
							    input_ids[:, -1:]     , attention_mask=UpperCAmelCase__     , past_key_values=outputs_cache.past_key_values     , position_ids=UpperCAmelCase__     , )

							snake_case   :					Tuple    						=				model(UpperCAmelCase__						)

							snake_case   :					Optional[int]    						=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
							self.parent.assertTrue(diff < 1e-3     , msg=F"Max diff is {diff}"						)



			def      lowerCAmelCase(   self   :  Dict     , UpperCAmelCase__   :  List[Any]     , UpperCAmelCase__   :  Union[str, Any]     , UpperCAmelCase__   :  Dict     , UpperCAmelCase__   :  Optional[int]						):






							"""simple docstring"""
							snake_case   :					Any    						=				20
							snake_case   :					Optional[int]    						=				model_class_name(UpperCAmelCase__						)

							snake_case   :					Optional[int]    						=				jnp.concatenate(
							    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])						)]     , axis=-1     , )

							snake_case   :					Dict    						=				model.init_cache(input_ids.shape[0]     , UpperCAmelCase__						)
							snake_case   :					Dict    						=				jnp.broadcast_to(
							    jnp.arange(input_ids.shape[-1] - 1						)[None, :]     , (input_ids.shape[0], input_ids.shape[-1] - 1)						)

							snake_case   :					str    						=				model(
							    input_ids[:, :-1]     , attention_mask=UpperCAmelCase__     , past_key_values=UpperCAmelCase__     , position_ids=UpperCAmelCase__     , )
							snake_case   :					Any    						=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     , dtype='''i4'''						)
							snake_case   :					Union[str, Any]    						=				model(
							    input_ids[:, -1:]     , past_key_values=outputs_cache.past_key_values     , attention_mask=UpperCAmelCase__     , position_ids=UpperCAmelCase__     , )

							snake_case   :					Any    						=				model(UpperCAmelCase__     , attention_mask=UpperCAmelCase__						)

							snake_case   :					Optional[int]    						=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
							self.parent.assertTrue(diff < 1e-3     , msg=F"Max diff is {diff}"						)
@require_flax
class   a_ (			a		,							a		,							unittest.TestCase      ):
			A__      :			Optional[int]       	=  (FlaxGPTJModel, FlaxGPTJForCausalLM) if is_flax_available() else ()
			A__      :			int       	=  (FlaxGPTJForCausalLM,) if is_flax_available() else ()

			def      lowerCAmelCase(   self   :  Union[str, Any]						):






							"""simple docstring"""
							snake_case   :					Optional[int]    						=				FlaxGPTJModelTester(self						)

			def      lowerCAmelCase(   self   :  List[Any]						):






							"""simple docstring"""


							for model_class_name in self.all_model_classes:
											snake_case						,						snake_case						,						snake_case   :					Dict    						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.check_use_cache_forward(UpperCAmelCase__     , UpperCAmelCase__     , UpperCAmelCase__     , UpperCAmelCase__						)

			def      lowerCAmelCase(   self   :  Any						):






							"""simple docstring"""


							for model_class_name in self.all_model_classes:
											snake_case						,						snake_case						,						snake_case   :					int    						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.check_use_cache_forward_with_attn_mask(
											    UpperCAmelCase__     , UpperCAmelCase__     , UpperCAmelCase__     , UpperCAmelCase__						)

			@tooslow
			def      lowerCAmelCase(   self   :  List[Any]						):






							"""simple docstring"""
							snake_case   :					Optional[Any]    						=				GPTaTokenizer.from_pretrained('''gpt2'''     , pad_token='''<|endoftext|>'''     , padding_side='''left'''						)
							snake_case   :					Dict    						=				tokenizer(['''Hello this is a long string''', '''Hey''']     , return_tensors='''np'''     , padding=UpperCAmelCase__     , truncation=UpperCAmelCase__						)

							snake_case   :					List[str]    						=				FlaxGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B'''						)
							snake_case   :					Tuple    						=				False
							snake_case   :					Union[str, Any]    						=				model.config.eos_token_id

							snake_case   :					Optional[Any]    						=				jax.jit(model.generate						)

							snake_case   :					List[Any]    						=				jit_generate(
							    inputs['''input_ids''']     , attention_mask=inputs['''attention_mask''']     , pad_token_id=tokenizer.pad_token_id						).sequences

							snake_case   :					Tuple    						=				tokenizer.batch_decode(UpperCAmelCase__     , skip_special_tokens=UpperCAmelCase__						)

							snake_case   :					Tuple    						=				[
							    '''Hello this is a long string of text.\n\nI\'m trying to get the text of the''',
							    '''Hey, I\'m a little late to the party. I\'m going to''',
							]

							self.assertListEqual(UpperCAmelCase__     , UpperCAmelCase__						)

			@is_pt_flax_cross_test
			def      lowerCAmelCase(   self   :  str						):






							"""simple docstring"""
							snake_case						,						snake_case   :					List[str]    						=				self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
											with self.subTest(model_class.__name__						):
															# prepare inputs
															snake_case   :					Optional[Any]    						=				self._prepare_for_class(UpperCAmelCase__     , UpperCAmelCase__						)
															snake_case   :					Tuple    						=				{k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

															# load corresponding PyTorch class
															snake_case   :					Any    						=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
															snake_case   :					Dict    						=				getattr(UpperCAmelCase__     , UpperCAmelCase__						)

															snake_case						,						snake_case   :					Dict    						=				pt_inputs['''input_ids'''].shape
															snake_case   :					Tuple    						=				np.random.randint(0     , seq_length - 1     , size=(batch_size,)						)
															for batch_idx, start_index in enumerate(UpperCAmelCase__						):
																			snake_case   :					str    						=				0
																			snake_case   :					Optional[int]    						=				1
																			snake_case   :					List[Any]    						=				0
																			snake_case   :					Union[str, Any]    						=				1
															snake_case   :					Optional[int]    						=				pt_model_class(UpperCAmelCase__						).eval()
															snake_case   :					List[Any]    						=				model_class(UpperCAmelCase__     , dtype=jnp.floataa						)

															snake_case   :					List[str]    						=				convert_pytorch_state_dict_to_flax(pt_model.state_dict()     , UpperCAmelCase__						)
															snake_case   :					List[str]    						=				fx_state

															with torch.no_grad():
																			snake_case   :					List[str]    						=				pt_model(**UpperCAmelCase__						).to_tuple()

															snake_case   :					int    						=				fx_model(**UpperCAmelCase__						).to_tuple()
															self.assertEqual(len(UpperCAmelCase__						)     , len(UpperCAmelCase__						)     , '''Output lengths differ between Flax and PyTorch'''						)
															for fx_output, pt_output in zip(UpperCAmelCase__     , UpperCAmelCase__						):
																			self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4e-2						)

															with tempfile.TemporaryDirectory() as tmpdirname:
																			pt_model.save_pretrained(UpperCAmelCase__						)
																			snake_case   :					Optional[Any]    						=				model_class.from_pretrained(UpperCAmelCase__     , from_pt=UpperCAmelCase__						)

															snake_case   :					Tuple    						=				fx_model_loaded(**UpperCAmelCase__						).to_tuple()
															self.assertEqual(
															    len(UpperCAmelCase__						)     , len(UpperCAmelCase__						)     , '''Output lengths differ between Flax and PyTorch'''						)
															for fx_output_loaded, pt_output in zip(UpperCAmelCase__     , UpperCAmelCase__						):
																			self.assert_almost_equals(fx_output_loaded[:, -1]     , pt_output[:, -1].numpy()     , 4e-2						)

			@is_pt_flax_cross_test
			def      lowerCAmelCase(   self   :  str						):






							"""simple docstring"""
							snake_case						,						snake_case   :					int    						=				self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
											with self.subTest(model_class.__name__						):
															# prepare inputs
															snake_case   :					int    						=				self._prepare_for_class(UpperCAmelCase__     , UpperCAmelCase__						)
															snake_case   :					Optional[Any]    						=				{k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

															# load corresponding PyTorch class
															snake_case   :					Optional[int]    						=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
															snake_case   :					Optional[int]    						=				getattr(UpperCAmelCase__     , UpperCAmelCase__						)

															snake_case   :					Union[str, Any]    						=				pt_model_class(UpperCAmelCase__						).eval()
															snake_case   :					int    						=				model_class(UpperCAmelCase__     , dtype=jnp.floataa						)

															snake_case   :					List[Any]    						=				load_flax_weights_in_pytorch_model(UpperCAmelCase__     , fx_model.params						)
															snake_case						,						snake_case   :					Union[str, Any]    						=				pt_inputs['''input_ids'''].shape
															snake_case   :					List[str]    						=				np.random.randint(0     , seq_length - 1     , size=(batch_size,)						)
															for batch_idx, start_index in enumerate(UpperCAmelCase__						):
																			snake_case   :					List[Any]    						=				0
																			snake_case   :					List[str]    						=				1
																			snake_case   :					List[Any]    						=				0
																			snake_case   :					int    						=				1

															# make sure weights are tied in PyTorch
															pt_model.tie_weights()

															with torch.no_grad():
																			snake_case   :					Tuple    						=				pt_model(**UpperCAmelCase__						).to_tuple()

															snake_case   :					str    						=				fx_model(**UpperCAmelCase__						).to_tuple()
															self.assertEqual(len(UpperCAmelCase__						)     , len(UpperCAmelCase__						)     , '''Output lengths differ between Flax and PyTorch'''						)
															for fx_output, pt_output in zip(UpperCAmelCase__     , UpperCAmelCase__						):
																			self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4e-2						)

															with tempfile.TemporaryDirectory() as tmpdirname:
																			fx_model.save_pretrained(UpperCAmelCase__						)
																			snake_case   :					Dict    						=				pt_model_class.from_pretrained(UpperCAmelCase__     , from_flax=UpperCAmelCase__						)

															with torch.no_grad():
																			snake_case   :					Dict    						=				pt_model_loaded(**UpperCAmelCase__						).to_tuple()

															self.assertEqual(
															    len(UpperCAmelCase__						)     , len(UpperCAmelCase__						)     , '''Output lengths differ between Flax and PyTorch'''						)
															for fx_output, pt_output in zip(UpperCAmelCase__     , UpperCAmelCase__						):
																			self.assert_almost_equals(fx_output[:, -1]     , pt_output[:, -1].numpy()     , 4e-2						)



			@tooslow
			def      lowerCAmelCase(   self   :  List[Any]						):






							"""simple docstring"""


							for model_class_name in self.all_model_classes:
											snake_case   :					List[str]    						=				model_class_name.from_pretrained('''EleutherAI/gpt-j-6B'''						)
											snake_case   :					Optional[int]    						=				model(np.ones((1, 1)						)						)
											self.assertIsNotNone(UpperCAmelCase__						)






'''simple docstring'''







import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCAmelCase_     						=				1_6
lowerCAmelCase_     						=				3_2







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase = 16						):






   '''simple docstring'''







   A__       =     AutoTokenizer.from_pretrained('bert-base-cased'						)
   A__       =     DatasetDict(
       {
           'train': dataset['train'].select(UpperCAmelCase						),
           'validation': dataset['train'].select(UpperCAmelCase						),
           'test': dataset['validation'],
       }						)

   def tokenize_function(UpperCAmelCase						):
      # max_length=None => use the model max length (it's actually the default)
      A__       =     tokenizer(examples['sentence1']      ,examples['sentence2']      ,truncation=UpperCAmelCase      ,max_length=UpperCAmelCase						)
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      A__       =     datasets.map(
          UpperCAmelCase      ,batched=UpperCAmelCase      ,remove_columns=['idx', 'sentence1', 'sentence2']      ,)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   A__       =     tokenized_datasets.rename_column('label'      ,'labels'						)

   def collate_fn(UpperCAmelCase						):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      A__       =     128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         A__       =     16
      elif accelerator.mixed_precision != "no":
         A__       =     8
      else:
         A__       =     None

      return tokenizer.pad(
          UpperCAmelCase      ,padding='longest'      ,max_length=UpperCAmelCase      ,pad_to_multiple_of=UpperCAmelCase      ,return_tensors='pt'      ,)

   # Instantiate dataloaders.
   A__       =     DataLoader(
       tokenized_datasets['train']      ,shuffle=UpperCAmelCase      ,collate_fn=UpperCAmelCase      ,batch_size=UpperCAmelCase						)
   A__       =     DataLoader(
       tokenized_datasets['validation']      ,shuffle=UpperCAmelCase      ,collate_fn=UpperCAmelCase      ,batch_size=UpperCAmelCase						)

   A__       =     DataLoader(
       tokenized_datasets['test']      ,shuffle=UpperCAmelCase      ,collate_fn=UpperCAmelCase      ,batch_size=UpperCAmelCase						)

   return train_dataloader, eval_dataloader, test_dataloader







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   A__       =     []
   # Download the dataset
   A__       =     load_dataset('glue'      ,'mrpc'						)
   # Create our splits
   A__       =     StratifiedKFold(n_splits=int(args.num_folds						)						)
   # Initialize accelerator
   A__       =     Accelerator(cpu=args.cpu      ,mixed_precision=args.mixed_precision						)
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   A__       =     config['lr']
   A__       =     int(config['num_epochs']						)
   A__       =     int(config['seed']						)
   A__       =     int(config['batch_size']						)

   A__       =     evaluate.load('glue'      ,'mrpc'						)

   # If the batch size is too big we use gradient accumulation
   A__       =     1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      A__       =     batch_size // MAX_GPU_BATCH_SIZE
      A__       =     MAX_GPU_BATCH_SIZE

   set_seed(UpperCAmelCase						)

   # New Code #
   # Create our folds:
   A__       =     kfold.split(np.zeros(datasets['train'].num_rows						)      ,datasets['train']['label']						)
   A__       =     []
   # Iterate over them
   for i, (train_idxs, valid_idxs) in enumerate(UpperCAmelCase						):
      A__  ,			A__  ,			A__       =     get_fold_dataloaders(
          UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,)
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      A__       =     AutoModelForSequenceClassification.from_pretrained('bert-base-cased'      ,return_dict=UpperCAmelCase						)

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      A__       =     model.to(accelerator.device						)

      # Instantiate optimizer
      A__       =     AdamW(params=model.parameters()      ,lr=UpperCAmelCase						)

      # Instantiate scheduler
      A__       =     get_linear_schedule_with_warmup(
          optimizer=UpperCAmelCase      ,num_warmup_steps=100      ,num_training_steps=(len(UpperCAmelCase						) * num_epochs) // gradient_accumulation_steps      ,)

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      A__  ,			A__  ,			A__  ,			A__  ,			A__       =     accelerator.prepare(
          UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)

      # Now we train the model
      for epoch in range(UpperCAmelCase						):
         model.train()
         for step, batch in enumerate(UpperCAmelCase						):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device						)
            A__       =     model(**UpperCAmelCase						)
            A__       =     outputs.loss
            A__       =     loss / gradient_accumulation_steps
            accelerator.backward(UpperCAmelCase						)
            if step % gradient_accumulation_steps == 0:
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()

         model.eval()
         for step, batch in enumerate(UpperCAmelCase						):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device						)
            with torch.no_grad():
               A__       =     model(**UpperCAmelCase						)
            A__       =     outputs.logits.argmax(dim=-1						)
            A__  ,			A__       =     accelerator.gather_for_metrics((predictions, batch['labels'])						)
            metric.add_batch(
                predictions=UpperCAmelCase      ,references=UpperCAmelCase      ,)

         A__       =     metric.compute()
         # Use accelerator.print to print only on the main process.
         accelerator.print(F"""epoch {epoch}:"""      ,UpperCAmelCase						)

      # New Code #
      # We also run predictions on the test set at the very end
      A__       =     []
      for step, batch in enumerate(UpperCAmelCase						):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device						)
         with torch.no_grad():
            A__       =     model(**UpperCAmelCase						)
         A__       =     outputs.logits
         A__  ,			A__       =     accelerator.gather_for_metrics((predictions, batch['labels'])						)
         fold_predictions.append(predictions.cpu()						)
         if i == 0:
            # We need all of the test predictions
            test_references.append(references.cpu()						)
        # Use accelerator.print to print only on the main process.
      test_predictions.append(torch.cat(UpperCAmelCase      ,dim=0						)						)
      # We now need to release all our memory and get rid of the current model, optimizer, etc
      accelerator.free_memory()
   # New Code #
   # Finally we check the accuracy of our folded results:
   A__       =     torch.cat(UpperCAmelCase      ,dim=0						)
   A__       =     torch.stack(UpperCAmelCase      ,dim=0						).sum(dim=0						).div(int(args.num_folds						)						).argmax(dim=-1						)
   A__       =     metric.compute(predictions=UpperCAmelCase      ,references=UpperCAmelCase						)
   accelerator.print('Average test metrics from all folds:'      ,UpperCAmelCase						)







def 						_A      ( ):






   '''simple docstring'''







   A__       =     argparse.ArgumentParser(description='Simple example of training script.'						)
   parser.add_argument(
       '--mixed_precision'      ,type=UpperCAmelCase      ,default=UpperCAmelCase      ,choices=['no', 'fp16', 'bf16', 'fp8']      ,help='Whether to use mixed precision. Choose'
       'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
       'and an Nvidia Ampere GPU.'      ,)
   parser.add_argument('--cpu'      ,action='store_true'      ,help='If passed, will train on the CPU.'						)
   # New Code #
   parser.add_argument('--num_folds'      ,type=UpperCAmelCase      ,default=3      ,help='The number of splits to perform across the dataset'						)
   A__       =     parser.parse_args()
   A__       =     {'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
   training_function(UpperCAmelCase      ,UpperCAmelCase						)


if __name__ == "__main__":
    main()


