





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


_lowerCamelCase      							=   get_tests_dir('fixtures')
_lowerCamelCase      							=   get_tests_dir('fixtures/dummy_feature_extractor_config.json')
_lowerCamelCase      							=   get_tests_dir('fixtures/dummy-config.json')







class       __A							(   unittest.TestCase	):




      """simple docstring"""







      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		0


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Tuple						=		AutoFeatureExtractor.from_pretrained('''facebook/wav2vec2-base-960h''')
             self.assertIsInstance(a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		AutoFeatureExtractor.from_pretrained(a__)
             self.assertIsInstance(a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase				:  Union[str, Any]						=		WavaVecaConfig()

                    # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
                    _lowerCamelCase				:  List[Any]						=		AutoFeatureExtractor.from_pretrained(a__).to_dict()

                    config_dict.pop('''feature_extractor_type''')
                    _lowerCamelCase				:  List[str]						=		WavaVecaFeatureExtractor(**a__)

                    # save in new folder
                    model_config.save_pretrained(a__)
                    config.save_pretrained(a__)

                    _lowerCamelCase				:  Dict						=		AutoFeatureExtractor.from_pretrained(a__)

                    # make sure private variable is not incorrectly saved
                    _lowerCamelCase				:  Tuple						=		json.loads(config.to_json_string())
                    self.assertTrue('''_processor_class''' not in dict_as_saved)

             self.assertIsInstance(a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		AutoFeatureExtractor.from_pretrained(a__)
             self.assertIsInstance(a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             with self.assertRaisesRegex(
                 a__   ,				'''bert-base is not a local folder and is not a valid model identifier'''):
                    _lowerCamelCase				:  List[Any]						=		AutoFeatureExtractor.from_pretrained('''bert-base''')


      def 					__snake_case      (		self):



             """simple docstring"""





             with self.assertRaisesRegex(
                 a__   ,				r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
                    _lowerCamelCase				:  Union[str, Any]						=		AutoFeatureExtractor.from_pretrained(a__   ,				revision='''aaaaaa''')


      def 					__snake_case      (		self):



             """simple docstring"""





             with self.assertRaisesRegex(
                 a__   ,				'''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''   ,				):
                    _lowerCamelCase				:  Union[str, Any]						=		AutoFeatureExtractor.from_pretrained('''hf-internal-testing/config-no-model''')


      def 					__snake_case      (		self):



             """simple docstring"""





             with self.assertRaises(a__):
                    _lowerCamelCase				:  str						=		AutoFeatureExtractor.from_pretrained(
                        '''hf-internal-testing/test_dynamic_feature_extractor''')
             # If remote code is disabled, we can't load this config.
             with self.assertRaises(a__):
                    _lowerCamelCase				:  int						=		AutoFeatureExtractor.from_pretrained(
                        '''hf-internal-testing/test_dynamic_feature_extractor'''   ,				trust_remote_code=a__)

             _lowerCamelCase				:  Dict						=		AutoFeatureExtractor.from_pretrained(
                 '''hf-internal-testing/test_dynamic_feature_extractor'''   ,				trust_remote_code=a__)
             self.assertEqual(feature_extractor.__class__.__name__   ,				'''NewFeatureExtractor''')

             # Test feature extractor can be reloaded.
             with tempfile.TemporaryDirectory() as tmp_dir:
                    feature_extractor.save_pretrained(a__)
                    _lowerCamelCase				:  Union[str, Any]						=		AutoFeatureExtractor.from_pretrained(a__   ,				trust_remote_code=a__)
             self.assertEqual(reloaded_feature_extractor.__class__.__name__   ,				'''NewFeatureExtractor''')


      def 					__snake_case      (		self):



             """simple docstring"""





             try:
                    AutoConfig.register('''custom'''   ,				a__)
                    AutoFeatureExtractor.register(a__   ,				a__)
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(a__):
                           AutoFeatureExtractor.register(a__   ,				a__)

                    # Now that the config is registered, it can be used as any other config with the auto-API
                    _lowerCamelCase				:  int						=		CustomFeatureExtractor.from_pretrained(a__)
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           feature_extractor.save_pretrained(a__)
                           _lowerCamelCase				:  Optional[int]						=		AutoFeatureExtractor.from_pretrained(a__)
                           self.assertIsInstance(a__   ,				a__)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]



      def 					__snake_case      (		self):



             """simple docstring"""





             class       __A							(   lowerCamelCase__	):




                   """simple docstring"""







                   UpperCAmelCase__							=   True

             try:
                    AutoConfig.register('''custom'''   ,				a__)
                    AutoFeatureExtractor.register(a__   ,				a__)
                    # If remote code is not set, the default is to use local
                    _lowerCamelCase				:  Optional[int]						=		AutoFeatureExtractor.from_pretrained(
                        '''hf-internal-testing/test_dynamic_feature_extractor''')
                    self.assertEqual(feature_extractor.__class__.__name__   ,				'''NewFeatureExtractor''')
                    self.assertTrue(feature_extractor.is_local)

                    # If remote code is disabled, we load the local one.
                    _lowerCamelCase				:  Tuple						=		AutoFeatureExtractor.from_pretrained(
                        '''hf-internal-testing/test_dynamic_feature_extractor'''   ,				trust_remote_code=a__)
                    self.assertEqual(feature_extractor.__class__.__name__   ,				'''NewFeatureExtractor''')
                    self.assertTrue(feature_extractor.is_local)

                    # If remote is enabled, we load from the Hub
                    _lowerCamelCase				:  Optional[Any]						=		AutoFeatureExtractor.from_pretrained(
                        '''hf-internal-testing/test_dynamic_feature_extractor'''   ,				trust_remote_code=a__)
                    self.assertEqual(feature_extractor.__class__.__name__   ,				'''NewFeatureExtractor''')
                    self.assertTrue(not hasattr(a__   ,				'''is_local'''))

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]








import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

