






import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures')
SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/dummy_feature_extractor_config.json')
SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/dummy-config.json')

class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				0


 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				AutoFeatureExtractor.from_pretrained('''facebook/wav2vec2-base-960h''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				AutoFeatureExtractor.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     with tempfile.TemporaryDirectory() as tmpdirname:
         _UpperCAmelCase				:      Tuple			  =				WavaVecaConfig()

         # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
         _UpperCAmelCase				:      List[Any]			  =				AutoFeatureExtractor.from_pretrained(_A).to_dict()

         config_dict.pop('''feature_extractor_type''')
         _UpperCAmelCase				:      Optional[int]			  =				WavaVecaFeatureExtractor(**_A)

         # save in new folder
         model_config.save_pretrained(_A)
         config.save_pretrained(_A)

         _UpperCAmelCase				:      Optional[Any]			  =				AutoFeatureExtractor.from_pretrained(_A)

         # make sure private variable is not incorrectly saved
         _UpperCAmelCase				:      List[Any]			  =				json.loads(config.to_json_string())
         self.assertTrue('''_processor_class''' not in dict_as_saved)

     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				AutoFeatureExtractor.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''bert-base is not a local folder and is not a valid model identifier'''):
         _UpperCAmelCase				:      List[Any]			  =				AutoFeatureExtractor.from_pretrained('''bert-base''')


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
         _UpperCAmelCase				:      Optional[Any]			  =				AutoFeatureExtractor.from_pretrained(_A							,  revision='''aaaaaa''')


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''							,  ):
         _UpperCAmelCase				:      int			  =				AutoFeatureExtractor.from_pretrained('''hf-internal-testing/config-no-model''')


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     with self.assertRaises(_A):
         _UpperCAmelCase				:      List[str]			  =				AutoFeatureExtractor.from_pretrained(
             '''hf-internal-testing/test_dynamic_feature_extractor''')
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(_A):
         _UpperCAmelCase				:      Optional[Any]			  =				AutoFeatureExtractor.from_pretrained(
             '''hf-internal-testing/test_dynamic_feature_extractor'''							,  trust_remote_code=_A)

     _UpperCAmelCase				:      Tuple			  =				AutoFeatureExtractor.from_pretrained(
         '''hf-internal-testing/test_dynamic_feature_extractor'''							,  trust_remote_code=_A)
     self.assertEqual(feature_extractor.__class__.__name__							,  '''NewFeatureExtractor''')

     # Test feature extractor can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
         feature_extractor.save_pretrained(_A)
         _UpperCAmelCase				:      Any			  =				AutoFeatureExtractor.from_pretrained(_A							,  trust_remote_code=_A)
     self.assertEqual(reloaded_feature_extractor.__class__.__name__							,  '''NewFeatureExtractor''')


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     try:
         AutoConfig.register('''custom'''							,  _A)
         AutoFeatureExtractor.register(_A							,  _A)
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(_A):
             AutoFeatureExtractor.register(_A							,  _A)

         # Now that the config is registered, it can be used as any other config with the auto-API
         _UpperCAmelCase				:      Optional[Any]			  =				CustomFeatureExtractor.from_pretrained(_A)
         with tempfile.TemporaryDirectory() as tmp_dir:
             feature_extractor.save_pretrained(_A)
             _UpperCAmelCase				:      Optional[int]			  =				AutoFeatureExtractor.from_pretrained(_A)
             self.assertIsInstance(_A							,  _A)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
             del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     class 							A_    (						__lowercase			):






      '''simple docstring'''




      _SCREAMING_SNAKE_CASE							:			List[str] =					True

     try:
         AutoConfig.register('''custom'''							,  _A)
         AutoFeatureExtractor.register(_A							,  _A)
         # If remote code is not set, the default is to use local
         _UpperCAmelCase				:      List[Any]			  =				AutoFeatureExtractor.from_pretrained(
             '''hf-internal-testing/test_dynamic_feature_extractor''')
         self.assertEqual(feature_extractor.__class__.__name__							,  '''NewFeatureExtractor''')
         self.assertTrue(feature_extractor.is_local)

         # If remote code is disabled, we load the local one.
         _UpperCAmelCase				:      Tuple			  =				AutoFeatureExtractor.from_pretrained(
             '''hf-internal-testing/test_dynamic_feature_extractor'''							,  trust_remote_code=_A)
         self.assertEqual(feature_extractor.__class__.__name__							,  '''NewFeatureExtractor''')
         self.assertTrue(feature_extractor.is_local)

         # If remote is enabled, we load from the Hub
         _UpperCAmelCase				:      int			  =				AutoFeatureExtractor.from_pretrained(
             '''hf-internal-testing/test_dynamic_feature_extractor'''							,  trust_remote_code=_A)
         self.assertEqual(feature_extractor.__class__.__name__							,  '''NewFeatureExtractor''')
         self.assertTrue(not hasattr(_A							,  '''is_local'''))

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
             del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]












import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							A_    (						__lowercase ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			int =					None
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			Any =					True
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					False
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					"tokenizer_file"
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     super().setUp()
     _UpperCAmelCase				:      List[Any]			  =				BloomTokenizerFast.from_pretrained('''bigscience/tokenizer''')
     tokenizer.save_pretrained(self.tmpdirname)


 def  snake_case__				(	self							,  **_A) ->		List[Any]:

     """simple docstring"""



     kwargs.update(self.special_tokens_map)
     return BloomTokenizerFast.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				self.get_rust_tokenizer()

     _UpperCAmelCase				:      int			  =				['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
     _UpperCAmelCase				:      List[Any]			  =				[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_encode_plus(_A)['''input_ids''']
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_decode(_A)
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self							,  _A=6) ->		Optional[Any]:

     """simple docstring"""



     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
             _UpperCAmelCase				:      Optional[int]			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)
             # tokenizer_r.pad_token = None # Hotfixing padding = None
             # Simple input
             _UpperCAmelCase				:      List[Any]			  =				'''This is a simple input'''
             _UpperCAmelCase				:      Any			  =				['''This is a simple input 1''', '''This is a simple input 2''']
             _UpperCAmelCase				:      int			  =				('''This is a simple input''', '''This is a pair''')
             _UpperCAmelCase				:      Optional[int]			  =				[
                 ('''This is a simple input 1''', '''This is a simple input 2'''),
                 ('''This is a simple pair 1''', '''This is a simple pair 2'''),
             ]

             # Simple input tests
             try:
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.encode_plus(_A							,  max_length=_A)

                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
             except ValueError:
                 self.fail('''Bloom Tokenizer should be able to deal with padding''')

             _UpperCAmelCase				:      Tuple			  =				None  # Hotfixing padding = None
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      int			  =				load_dataset('''xnli'''							,  '''all_languages'''							,  split='''test'''							,  streaming=_A)

     _UpperCAmelCase				:      Tuple			  =				next(iter(_A))['''premise''']  # pick up one data
     _UpperCAmelCase				:      List[Any]			  =				list(sample_data.values())

     _UpperCAmelCase				:      Any			  =				list(map(tokenizer.encode							,  _A))
     _UpperCAmelCase				:      List[str]			  =				[tokenizer.decode(_A							,  clean_up_tokenization_spaces=_A) for x in output_tokens]
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)							,  1)
     self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])							,  1)




