





'''simple docstring'''


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a__			:					List[str]       						=						'▁'

a__			:					str       						=						get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class     lowerCAmelCase__ (				UpperCAmelCase_      , unittest.TestCase    ):

       '''simple docstring'''
       _lowerCamelCase								=BigBirdTokenizer
       _lowerCamelCase								=BigBirdTokenizerFast
       _lowerCamelCase								=True
       _lowerCamelCase								=True

       def 			__snake_case      ( self					:				Union[str, Any]						):
              super().setUp()

              UpperCAmelCase  =						self.tokenizer_class(_UpperCamelCase       ,     keep_accents=_UpperCamelCase						)
              tokenizer.save_pretrained(self.tmpdirname						)

       def 			__snake_case      ( self					:				Dict						):
              UpperCAmelCase  =						'''<s>'''
              UpperCAmelCase  =						1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase						)       ,     _UpperCamelCase						)
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase						)       ,     _UpperCamelCase						)

       def 			__snake_case      ( self					:				Tuple						):
              UpperCAmelCase  =						list(self.get_tokenizer().get_vocab().keys()						)

              self.assertEqual(vocab_keys[0]       ,     '''<unk>'''						)
              self.assertEqual(vocab_keys[1]       ,     '''<s>'''						)
              self.assertEqual(vocab_keys[-1]       ,     '''[MASK]'''						)
              self.assertEqual(len(_UpperCamelCase						)       ,     1004						)

       def 			__snake_case      ( self					:				Dict						):
              self.assertEqual(self.get_tokenizer().vocab_size       ,     1000						)

       def 			__snake_case      ( self					:				str						):
              if not self.test_rust_tokenizer:
                     return

              UpperCAmelCase  =						self.get_tokenizer()
              UpperCAmelCase  =						self.get_rust_tokenizer()

              UpperCAmelCase  =						'''I was born in 92000, and this is falsé.'''

              UpperCAmelCase  =						tokenizer.tokenize(_UpperCamelCase						)
              UpperCAmelCase  =						rust_tokenizer.tokenize(_UpperCamelCase						)
              self.assertListEqual(_UpperCamelCase       ,     _UpperCamelCase						)

              UpperCAmelCase  =						tokenizer.encode(_UpperCamelCase       ,     add_special_tokens=_UpperCamelCase						)
              UpperCAmelCase  =						rust_tokenizer.encode(_UpperCamelCase       ,     add_special_tokens=_UpperCamelCase						)
              self.assertListEqual(_UpperCamelCase       ,     _UpperCamelCase						)

              UpperCAmelCase  =						self.get_rust_tokenizer()
              UpperCAmelCase  =						tokenizer.encode(_UpperCamelCase						)
              UpperCAmelCase  =						rust_tokenizer.encode(_UpperCamelCase						)
              self.assertListEqual(_UpperCamelCase       ,     _UpperCamelCase						)

       def 			__snake_case      ( self					:				Any						):
              UpperCAmelCase  =						BigBirdTokenizer(_UpperCamelCase       ,     keep_accents=_UpperCamelCase						)

              UpperCAmelCase  =						tokenizer.tokenize('''This is a test'''						)
              self.assertListEqual(_UpperCamelCase       ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']						)

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(_UpperCamelCase						)       ,     [285, 46, 10, 170, 382]       ,     )

              UpperCAmelCase  =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''						)
              self.assertListEqual(
                  _UpperCamelCase       ,     [
                      SPIECE_UNDERLINE + '''I''',
                      SPIECE_UNDERLINE + '''was''',
                      SPIECE_UNDERLINE + '''b''',
                      '''or''',
                      '''n''',
                      SPIECE_UNDERLINE + '''in''',
                      SPIECE_UNDERLINE + '''''',
                      '''9''',
                      '''2''',
                      '''0''',
                      '''0''',
                      '''0''',
                      ''',''',
                      SPIECE_UNDERLINE + '''and''',
                      SPIECE_UNDERLINE + '''this''',
                      SPIECE_UNDERLINE + '''is''',
                      SPIECE_UNDERLINE + '''f''',
                      '''al''',
                      '''s''',
                      '''é''',
                      '''.''',
                  ]       ,     )
              UpperCAmelCase  =						tokenizer.convert_tokens_to_ids(_UpperCamelCase						)
              self.assertListEqual(
                  _UpperCamelCase       ,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,     )

              UpperCAmelCase  =						tokenizer.convert_ids_to_tokens(_UpperCamelCase						)
              self.assertListEqual(
                  _UpperCamelCase       ,     [
                      SPIECE_UNDERLINE + '''I''',
                      SPIECE_UNDERLINE + '''was''',
                      SPIECE_UNDERLINE + '''b''',
                      '''or''',
                      '''n''',
                      SPIECE_UNDERLINE + '''in''',
                      SPIECE_UNDERLINE + '''''',
                      '''<unk>''',
                      '''2''',
                      '''0''',
                      '''0''',
                      '''0''',
                      ''',''',
                      SPIECE_UNDERLINE + '''and''',
                      SPIECE_UNDERLINE + '''this''',
                      SPIECE_UNDERLINE + '''is''',
                      SPIECE_UNDERLINE + '''f''',
                      '''al''',
                      '''s''',
                      '''<unk>''',
                      '''.''',
                  ]       ,     )

       @cached_property
       def 			__snake_case      ( self					:				Tuple						):
              return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''						)

       @slow
       def 			__snake_case      ( self					:				str						):
              UpperCAmelCase  =						'''Hello World!'''
              UpperCAmelCase  =						[65, 18536, 2260, 101, 66]

              self.assertListEqual(_UpperCamelCase       ,     self.big_tokenizer.encode(_UpperCamelCase						)						)

       @slow
       def 			__snake_case      ( self					:				Optional[Any]						):
              UpperCAmelCase  =						(
                  '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
                  ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
              )
              # fmt: off
              UpperCAmelCase  =						[65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
              # fmt: on
              self.assertListEqual(_UpperCamelCase       ,     self.big_tokenizer.encode(_UpperCamelCase						)						)

       @require_torch
       @slow
       def 			__snake_case      ( self					:				Optional[int]						):
              import torch

              from transformers import BigBirdConfig, BigBirdModel

              # Build sequence
              UpperCAmelCase  =						list(self.big_tokenizer.get_vocab().keys()						)[:10]
              UpperCAmelCase  =						''' '''.join(_UpperCamelCase						)
              UpperCAmelCase  =						self.big_tokenizer.encode_plus(_UpperCamelCase       ,     return_tensors='''pt'''       ,     return_token_type_ids=_UpperCamelCase						)
              UpperCAmelCase  =						self.big_tokenizer.batch_encode_plus(
                  [sequence + ''' ''' + sequence]       ,     return_tensors='''pt'''       ,     return_token_type_ids=_UpperCamelCase						)

              UpperCAmelCase  =						BigBirdConfig(attention_type='''original_full'''						)
              UpperCAmelCase  =						BigBirdModel(_UpperCamelCase						)

              assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

              with torch.no_grad():
                     model(**_UpperCamelCase						)
                     model(**_UpperCamelCase						)

       @slow
       def 			__snake_case      ( self					:				Optional[int]						):
              UpperCAmelCase  =						BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''						)
              UpperCAmelCase  =						tokenizer.decode(tokenizer('''Paris is the [MASK].'''						).input_ids						)

              self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]'''						)



       @slow
       def 			__snake_case      ( self					:				Optional[int]						):
              # fmt: off
              UpperCAmelCase  =						{'''input_ids''': [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=_UpperCamelCase       ,     model_name='''google/bigbird-roberta-base'''       ,     revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''       ,     )






'''simple docstring'''
import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ASTForAudioClassification, ASTModel
    from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
        AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
    )


if is_torchaudio_available():
    import torchaudio

    from transformers import ASTFeatureExtractor






class        lowerCAmelCase_  :






       def __init__( self			,     _UpperCamelCase			,     _UpperCamelCase=13			,     _UpperCamelCase=2			,     _UpperCamelCase=24			,     _UpperCamelCase=16			,     _UpperCamelCase=True			,     _UpperCamelCase=True			,     _UpperCamelCase=32			,     _UpperCamelCase=5			,     _UpperCamelCase=4			,     _UpperCamelCase=37			,     _UpperCamelCase="gelu"			,     _UpperCamelCase=0.1			,     _UpperCamelCase=0.1			,     _UpperCamelCase=10			,     _UpperCamelCase=0.02			,     _UpperCamelCase=None			,     _UpperCamelCase=2			,     _UpperCamelCase=2			,     )->      Tuple:
            _A				=							parent
            _A				=							batch_size
            _A				=							patch_size
            _A				=							max_length
            _A				=							num_mel_bins
            _A				=							is_training
            _A				=							use_labels
            _A				=							hidden_size
            _A				=							num_hidden_layers
            _A				=							num_attention_heads
            _A				=							intermediate_size
            _A				=							hidden_act
            _A				=							hidden_dropout_prob
            _A				=							attention_probs_dropout_prob
            _A				=							type_sequence_label_size
            _A				=							initializer_range
            _A				=							scope
            _A				=							frequency_stride
            _A				=							time_stride

            # in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
            _A				=							(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
            _A				=							(self.max_length - self.patch_size) // self.time_stride + 1
            _A				=							frequency_out_dimension * time_out_dimension
            _A				=							num_patches + 2






       def   UpperCamelCase						( self   )->      int:
            _A				=							floats_tensor([self.batch_size, self.max_length, self.num_mel_bins]   )

            _A				=							None
            if self.use_labels:
                 _A				=							ids_tensor([self.batch_size]			,     self.type_sequence_label_size   )

            _A				=							self.get_config()

            return config, input_values, labels






       def   UpperCamelCase						( self   )->      int:
            return ASTConfig(
                patch_size=self.patch_size			,     max_length=self.max_length			,     num_mel_bins=self.num_mel_bins			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     is_decoder=_UpperCamelCase			,     initializer_range=self.initializer_range			,     frequency_stride=self.frequency_stride			,     time_stride=self.time_stride			,     )






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )->      Optional[Any]:
            _A				=							ASTModel(config=_UpperCamelCase   )
            model.to(_UpperCamelCase   )
            model.eval()
            _A				=							model(_UpperCamelCase   )
            self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )






       def   UpperCamelCase						( self   )->      Any:
            _A				=							self.prepare_config_and_inputs()
            (
                (
                _A
            )		,				(
                _A
            )		,				(
                _A
            )		,				
            )				=							config_and_inputs
            _A				=							{'input_values': input_values}
            return config, inputs_dict






@require_torch
class        lowerCAmelCase_  (     UpperCAmelCase      ,							UpperCAmelCase      ,							unittest.TestCase	):
       __UpperCAmelCase					=(
           (
               ASTModel,
               ASTForAudioClassification,
           )
           if is_torch_available()
           else ()
       )
       __UpperCAmelCase					=(
           {'audio-classification': ASTForAudioClassification, 'feature-extraction': ASTModel}
           if is_torch_available()
           else {}
       )
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False
       __UpperCAmelCase					=False






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )->      Union[str, Any]:
            if pipeline_test_casse_name == "AudioClassificationPipelineTests":
                 return True

            return False






       def   UpperCamelCase						( self   )->      List[str]:
            _A				=							ASTModelTester(self   )
            _A				=							ConfigTester(self			,     config_class=_UpperCamelCase			,     has_text_modality=_UpperCamelCase			,     hidden_size=37   )






       def   UpperCamelCase						( self   )->      int:
            self.config_tester.run_common_tests()






       @unittest.skip(reason='AST does not use inputs_embeds'   )
       def   UpperCamelCase						( self   )->      List[Any]:
            pass






       def   UpperCamelCase						( self   )->      str:
            _A		,				_A				=							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 _A				=							model_class(_UpperCamelCase   )
                 self.assertIsInstance(model.get_input_embeddings()			,     (nn.Module)   )
                 _A				=							model.get_output_embeddings()
                 self.assertTrue(x is None or isinstance(_UpperCamelCase			,     nn.Linear   )   )






       def   UpperCamelCase						( self   )->      List[str]:
            _A		,				_A				=							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                 _A				=							model_class(_UpperCamelCase   )
                 _A				=							inspect.signature(model.forward   )
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 _A				=							[*signature.parameters.keys()]

                 _A				=							['input_values']
                 self.assertListEqual(arg_names[:1]			,     _UpperCamelCase   )






       def   UpperCamelCase						( self   )->      Optional[Any]:
            _A				=							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*_UpperCamelCase   )






       @slow
       def   UpperCamelCase						( self   )->      Tuple:
            for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 _A				=							ASTModel.from_pretrained(_UpperCamelCase   )
                 self.assertIsNotNone(_UpperCamelCase   )






def 					lowerCamelCase_ ( )					->	Union[str, Any]:







     """simple docstring"""
     _A				=							hf_hub_download(
         repo_id='nielsr/audio-spectogram-transformer-checkpoint'							,    filename='sample_audio.flac'							,    repo_type='dataset'		)

     _A		,				_A				=							torchaudio.load(__UpperCamelCase		)

     return audio, sampling_rate






@require_torch
@require_torchaudio
class        lowerCAmelCase_  (     unittest.TestCase	):






       @cached_property
       def   UpperCamelCase						( self   )->      Dict:
            return (
                ASTFeatureExtractor.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'   )
                if is_torchaudio_available()
                else None
            )






       @slow
       def   UpperCamelCase						( self   )->      Any:
            _A				=							self.default_feature_extractor
            _A				=							ASTForAudioClassification.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'   ).to(_UpperCamelCase   )

            _A				=							self.default_feature_extractor
            _A		,				_A				=							prepare_audio()
            _A				=							audio.squeeze().numpy()
            _A				=							feature_extractor(_UpperCamelCase			,     sampling_rate=_UpperCamelCase			,     return_tensors='pt'   ).to(_UpperCamelCase   )

            # forward pass
            with torch.no_grad():
                 _A				=							model(**_UpperCamelCase   )

            # verify the logits
            _A				=							torch.Size((1, 527)   )
            self.assertEqual(outputs.logits.shape			,     _UpperCamelCase   )

            _A				=							torch.tensor([-0.8760, -7.0042, -8.6602]   ).to(_UpperCamelCase   )

            self.assertTrue(torch.allclose(outputs.logits[0, :3]			,     _UpperCamelCase			,     atol=1e-4   )   )





