





"""simple docstring"""






import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A			      =	16
A			      =	32

def      __A   (      a_    :Accelerator ,							a_    :int = 16)		->		Any:
				__a			:    Dict      							=	AutoTokenizer.from_pretrained('''bert-base-cased''')
				__a			:    List[str]      							=	load_dataset('''glue''' ,							'''mrpc''')

				def tokenize_function(a_    :List[str]):
								# max_length=None => use the model max length (it's actually the default)
								__a			:    List[Any]      							=	tokenizer(examples['''sentence1'''] ,							examples['''sentence2'''] ,							truncation=a_ ,							max_length=a_)
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				# starting with the main process first:
				with accelerator.main_process_first():
								__a			:    str      							=	datasets.map(
								    a_ ,							batched=a_ ,							remove_columns=['''idx''', '''sentence1''', '''sentence2'''] ,							)

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				__a			:    Optional[Any]      							=	tokenized_datasets.rename_column('''label''' ,							'''labels''')

				def collate_fn(a_    :Dict):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								__a			:    Tuple      							=	1_28 if accelerator.distributed_type == DistributedType.TPU else None
								# When using mixed precision we want round multiples of 8/16
								if accelerator.mixed_precision == "fp8":
												__a			:    Union[str, Any]      							=	16
								elif accelerator.mixed_precision != "no":
												__a			:    int      							=	8
								else:
												__a			:    Tuple      							=	None

								return tokenizer.pad(
								    a_ ,							padding='''longest''' ,							max_length=a_ ,							pad_to_multiple_of=a_ ,							return_tensors='''pt''' ,							)

				# Instantiate dataloaders.
				__a			:    Optional[Any]      							=	DataLoader(
				    tokenized_datasets['''train'''] ,							shuffle=a_ ,							collate_fn=a_ ,							batch_size=a_)
				__a			:    List[str]      							=	DataLoader(
				    tokenized_datasets['''validation'''] ,							shuffle=a_ ,							collate_fn=a_ ,							batch_size=a_)

				return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		A			      =	mocked_dataloaders  # noqa: F811

def      __A   (      a_    :Union[str, Any] ,							a_    :str)		->		Union[str, Any]:
				# For testing only
				if os.environ.get('''TESTING_MOCKED_DATALOADERS''' ,							a_) == "1":
								__a			:    List[str]      							=	2
				# Initialize accelerator
				__a			:    Union[str, Any]      							=	Accelerator(cpu=args.cpu ,							mixed_precision=args.mixed_precision)
				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				__a			:    Any      							=	config['''lr''']
				__a			:    Dict      							=	int(config['''num_epochs'''])
				__a			:    List[str]      							=	int(config['''seed'''])
				__a			:    Union[str, Any]      							=	int(config['''batch_size'''])

				__a			:    Tuple      							=	evaluate.load('''glue''' ,							'''mrpc''')

				# New Code #
				# We now can define an inner training loop function. It should take a batch size as the only parameter,
				# and build the dataloaders in there.
				# It also gets our decorator
				@find_executable_batch_size(starting_batch_size=a_)
				def inner_training_loop(a_    :List[str]):
								# And now just move everything below under this function
								# We need to bring in the Accelerator object from earlier
								nonlocal accelerator
								# And reset all of its attributes that could hold onto any memory:
								accelerator.free_memory()

								# Then we can declare the model, optimizer, and everything else:
								set_seed(a_)

								# Instantiate the model (we build the model here so that the seed also control new weights initialization)
								__a			:    Tuple      							=	AutoModelForSequenceClassification.from_pretrained('''bert-base-cased''' ,							return_dict=a_)

								# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
								# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
								# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
								__a			:    str      							=	model.to(accelerator.device)

								# Instantiate optimizer
								__a			:    int      							=	AdamW(params=model.parameters() ,							lr=a_)
								__a							,    __a			:    int      							=	get_dataloaders(a_ ,							a_)

								# Instantiate scheduler
								__a			:    Any      							=	get_linear_schedule_with_warmup(
								    optimizer=a_ ,							num_warmup_steps=1_00 ,							num_training_steps=(len(a_) * num_epochs) ,							)

								# Prepare everything
								# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
								# prepare method.
								__a							,    __a							,    __a							,    __a							,    __a			:    str      							=	accelerator.prepare(
								    a_ ,							a_ ,							a_ ,							a_ ,							a_)

								# Now we train the model
								for epoch in range(a_):
												model.train()
												for step, batch in enumerate(a_):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device)
																__a			:    str      							=	model(**a_)
																__a			:    Optional[Any]      							=	outputs.loss
																accelerator.backward(a_)
																optimizer.step()
																lr_scheduler.step()
																optimizer.zero_grad()

												model.eval()
												for step, batch in enumerate(a_):
																# We could avoid this line since we set the accelerator with `device_placement=True`.
																batch.to(accelerator.device)
																with torch.no_grad():
																				__a			:    List[str]      							=	model(**a_)
																__a			:    List[str]      							=	outputs.logits.argmax(dim=-1)
																__a							,    __a			:    List[str]      							=	accelerator.gather_for_metrics((predictions, batch['''labels''']))
																metric.add_batch(
																    predictions=a_ ,							references=a_ ,							)

												__a			:    Tuple      							=	metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(F"""epoch {epoch}:""" ,							a_)

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
				inner_training_loop()

def      __A   (      )		->		str:
				__a			:    Tuple      							=	argparse.ArgumentParser(description='''Simple example of training script.''')
				parser.add_argument(
				    '''--mixed_precision''' ,							type=a_ ,							default=a_ ,							choices=['''no''', '''fp16''', '''bf16''', '''fp8'''] ,							help='''Whether to use mixed precision. Choose'''
				    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
				    '''and an Nvidia Ampere GPU.''' ,							)
				parser.add_argument('''--cpu''' ,							action='''store_true''' ,							help='''If passed, will train on the CPU.''')
				__a			:    Optional[Any]      							=	parser.parse_args()
				__a			:    int      							=	{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
				training_function(a_ ,							a_)


if __name__ == "__main__":
		main()



'''simple docstring'''
import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ASTForAudioClassification, ASTModel
    from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
        AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
    )


if is_torchaudio_available():
    import torchaudio

    from transformers import ASTFeatureExtractor






class        lowerCAmelCase_  :






       def __init__( self			,     _UpperCamelCase			,     _UpperCamelCase=13			,     _UpperCamelCase=2			,     _UpperCamelCase=24			,     _UpperCamelCase=16			,     _UpperCamelCase=True			,     _UpperCamelCase=True			,     _UpperCamelCase=32			,     _UpperCamelCase=5			,     _UpperCamelCase=4			,     _UpperCamelCase=37			,     _UpperCamelCase="gelu"			,     _UpperCamelCase=0.1			,     _UpperCamelCase=0.1			,     _UpperCamelCase=10			,     _UpperCamelCase=0.02			,     _UpperCamelCase=None			,     _UpperCamelCase=2			,     _UpperCamelCase=2			,     )->      Tuple:
            _A				=							parent
            _A				=							batch_size
            _A				=							patch_size
            _A				=							max_length
            _A				=							num_mel_bins
            _A				=							is_training
            _A				=							use_labels
            _A				=							hidden_size
            _A				=							num_hidden_layers
            _A				=							num_attention_heads
            _A				=							intermediate_size
            _A				=							hidden_act
            _A				=							hidden_dropout_prob
            _A				=							attention_probs_dropout_prob
            _A				=							type_sequence_label_size
            _A				=							initializer_range
            _A				=							scope
            _A				=							frequency_stride
            _A				=							time_stride

            # in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
            _A				=							(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
            _A				=							(self.max_length - self.patch_size) // self.time_stride + 1
            _A				=							frequency_out_dimension * time_out_dimension
            _A				=							num_patches + 2






       def   UpperCamelCase						( self   )->      int:
            _A				=							floats_tensor([self.batch_size, self.max_length, self.num_mel_bins]   )

            _A				=							None
            if self.use_labels:
                 _A				=							ids_tensor([self.batch_size]			,     self.type_sequence_label_size   )

            _A				=							self.get_config()

            return config, input_values, labels






       def   UpperCamelCase						( self   )->      int:
            return ASTConfig(
                patch_size=self.patch_size			,     max_length=self.max_length			,     num_mel_bins=self.num_mel_bins			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     is_decoder=_UpperCamelCase			,     initializer_range=self.initializer_range			,     frequency_stride=self.frequency_stride			,     time_stride=self.time_stride			,     )






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )->      Optional[Any]:
            _A				=							ASTModel(config=_UpperCamelCase   )
            model.to(_UpperCamelCase   )
            model.eval()
            _A				=							model(_UpperCamelCase   )
            self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )






       def   UpperCamelCase						( self   )->      Any:
            _A				=							self.prepare_config_and_inputs()
            (
                (
                _A
            )		,				(
                _A
            )		,				(
                _A
            )		,				
            )				=							config_and_inputs
            _A				=							{'input_values': input_values}
            return config, inputs_dict






@require_torch
class        lowerCAmelCase_  (     UpperCAmelCase      ,							UpperCAmelCase      ,							unittest.TestCase	):
       __UpperCAmelCase					=(
           (
               ASTModel,
               ASTForAudioClassification,
           )
           if is_torch_available()
           else ()
       )
       __UpperCAmelCase					=(
           {'audio-classification': ASTForAudioClassification, 'feature-extraction': ASTModel}
           if is_torch_available()
           else {}
       )
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )->      Union[str, Any]:
            if pipeline_test_casse_name == "AudioClassificationPipelineTests":
                 return True

            return False






       def   UpperCamelCase						( self   )->      List[str]:
            _A				=							ASTModelTester(self   )
            _A				=							ConfigTester(self			,     config_class=_UpperCamelCase			,     has_text_modality=_UpperCamelCase			,     hidden_size=37   )






       def   UpperCamelCase						( self   )->      int:
            self.config_tester.run_common_tests()






       @unittest.skip(reason='AST does not use inputs_embeds'   )
       def   UpperCamelCase						( self   )->      List[Any]:
            pass






       def   UpperCamelCase						( self   )->      str:
            _A		,				_A				=							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 _A				=							model_class(_UpperCamelCase   )
                 self.assertIsInstance(model.get_input_embeddings()			,     (nn.Module)   )
                 _A				=							model.get_output_embeddings()
                 self.assertTrue(x is None or isinstance(_UpperCamelCase			,     nn.Linear   )   )






       def   UpperCamelCase						( self   )->      List[str]:
            _A		,				_A				=							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 _A				=							model_class(_UpperCamelCase   )
                 _A				=							inspect.signature(model.forward   )
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 _A				=							[*signature.parameters.keys()]

                 _A				=							['input_values']
                 self.assertListEqual(arg_names[:1]			,     _UpperCamelCase   )






       def   UpperCamelCase						( self   )->      Optional[Any]:
            _A				=							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*_UpperCamelCase   )






       @slow
       def   UpperCamelCase						( self   )->      Tuple:
            for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 _A				=							ASTModel.from_pretrained(_UpperCamelCase   )
                 self.assertIsNotNone(_UpperCamelCase   )






def 					lowerCamelCase_ ( )					->	Union[str, Any]:







     """simple docstring"""
     _A				=							hf_hub_download(
         repo_id='nielsr/audio-spectogram-transformer-checkpoint'							,    filename='sample_audio.flac'							,    repo_type='dataset'		)

     _A		,				_A				=							torchaudio.load(__UpperCamelCase		)

     return audio, sampling_rate






@require_torch
@require_torchaudio
class        lowerCAmelCase_  (     unittest.TestCase	):






       @cached_property
       def   UpperCamelCase						( self   )->      Dict:
            return (
                ASTFeatureExtractor.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'   )
                if is_torchaudio_available()
                else None
            )






       @slow
       def   UpperCamelCase						( self   )->      Any:
            _A				=							self.default_feature_extractor
            _A				=							ASTForAudioClassification.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'   ).to(_UpperCamelCase   )

            _A				=							self.default_feature_extractor
            _A		,				_A				=							prepare_audio()
            _A				=							audio.squeeze().numpy()
            _A				=							feature_extractor(_UpperCamelCase			,     sampling_rate=_UpperCamelCase			,     return_tensors='pt'   ).to(_UpperCamelCase   )

            # forward pass
            with torch.no_grad():
                 _A				=							model(**_UpperCamelCase   )

            # verify the logits
            _A				=							torch.Size((1, 527)   )
            self.assertEqual(outputs.logits.shape			,     _UpperCamelCase   )

            _A				=							torch.tensor([-0.8760, -7.0042, -8.6602]   ).to(_UpperCamelCase   )

            self.assertTrue(torch.allclose(outputs.logits[0, :3]			,     _UpperCamelCase			,     atol=1e-4   )   )





