
'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE     =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE     =       {
    """vocab_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"""
        ),
        """squeezebert/squeezebert-mnli""": """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt""",
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE     =       {
    """squeezebert/squeezebert-uncased""": 5_1_2,
    """squeezebert/squeezebert-mnli""": 5_1_2,
    """squeezebert/squeezebert-mnli-headless""": 5_1_2,
}


SCREAMING_SNAKE_CASE     =       {
    """squeezebert/squeezebert-uncased""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli-headless""": {"""do_lower_case""": True},
}






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    Dict				=							VOCAB_FILES_NAMES
 lowercase_					:    List[str]				=							PRETRAINED_VOCAB_FILES_MAP
 lowercase_					:    int				=							PRETRAINED_INIT_CONFIGURATION
 lowercase_					:    List[Any]				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_					:    str				=							SqueezeBertTokenizer







 def __init__(					self		:  Tuple			,				snake_case__		:  Tuple=None			,				snake_case__		:  Optional[int]=None			,				snake_case__		:  Tuple=True			,				snake_case__		:  Tuple="[UNK]"			,				snake_case__		:  Dict="[SEP]"			,				snake_case__		:  Union[str, Any]="[PAD]"			,				snake_case__		:  Union[str, Any]="[CLS]"			,				snake_case__		:  List[str]="[MASK]"			,				snake_case__		:  int=True			,				snake_case__		:  str=None			,				**snake_case__		:  Dict			,				):



   '''simple docstring'''




   super().__init__(
       snake_case__			,				tokenizer_file=snake_case__			,				do_lower_case=snake_case__			,				unk_token=snake_case__			,				sep_token=snake_case__			,				pad_token=snake_case__			,				cls_token=snake_case__			,				mask_token=snake_case__			,				tokenize_chinese_chars=snake_case__			,				strip_accents=snake_case__			,				**snake_case__			,				)

   UpperCAmelCase__      :					Any				       =							json.loads(self.backend_tokenizer.normalizer.__getstate__() )
   if (
       normalizer_state.get("lowercase"			,				snake_case__ ) != do_lower_case
       or normalizer_state.get("strip_accents"			,				snake_case__ ) != strip_accents
       or normalizer_state.get("handle_chinese_chars"			,				snake_case__ ) != tokenize_chinese_chars
   ):
     UpperCAmelCase__      :					Any				       =							getattr(snake_case__			,				normalizer_state.pop("type" ) )
     UpperCAmelCase__      :					Tuple				       =							do_lower_case
     UpperCAmelCase__      :					List[Any]				       =							strip_accents
     UpperCAmelCase__      :					Optional[Any]				       =							tokenize_chinese_chars
     UpperCAmelCase__      :					List[str]				       =							normalizer_class(**snake_case__ )

   UpperCAmelCase__      :					str				       =							do_lower_case







 def 							UpperCamelCase  (					self		:  Any			,				snake_case__		:  int			,				snake_case__		:  List[Any]=None ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  List[int]			,				snake_case__		:  Optional[List[int]] = None ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							[self.sep_token_id]
   UpperCAmelCase__      :					str				       =							[self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep ) * [0]
   return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







 def 							UpperCamelCase  (					self		:  Any			,				snake_case__		:  str			,				snake_case__		:  Optional[str] = None ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							self._tokenizer.model.save(snake_case__			,				name=snake_case__ )
   return tuple(snake_case__ )





'''simple docstring'''





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures""")
SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures/dummy-config.json""")






class 							UpperCAmelCase_							(    unittest.TestCase				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							0







 def 							UpperCamelCase  (					self		:  Union[str, Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h" )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   with tempfile.TemporaryDirectory() as tmpdirname:
     UpperCAmelCase__      :					Optional[Any]				       =							WavaVecaConfig()

     # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__ ).to_dict()

     config_dict.pop("feature_extractor_type" )
     UpperCAmelCase__      :					Union[str, Any]				       =							WavaVecaFeatureExtractor(**snake_case__ )

     # save in new folder
     model_config.save_pretrained(snake_case__ )
     config.save_pretrained(snake_case__ )

     UpperCAmelCase__      :					Tuple				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )

     # make sure private variable is not incorrectly saved
     UpperCAmelCase__      :					List[Any]				       =							json.loads(config.to_json_string() )
     self.assertTrue("_processor_class" not in dict_as_saved )

   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
   self.assertIsInstance(snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				"bert-base is not a local folder and is not a valid model identifier" ):
     UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained("bert-base" )







 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)" ):
     UpperCAmelCase__      :					List[str]				       =							AutoFeatureExtractor.from_pretrained(snake_case__			,				revision="aaaaaa" )







 def 							UpperCamelCase  (					self		:  Union[str, Any] ):



   '''simple docstring'''




   with self.assertRaisesRegex(
       snake_case__			,				"hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."			,				):
     UpperCAmelCase__      :					Optional[Any]				       =							AutoFeatureExtractor.from_pretrained("hf-internal-testing/config-no-model" )







 def 							UpperCamelCase  (					self		:  Optional[Any] ):



   '''simple docstring'''




   with self.assertRaises(snake_case__ ):
     UpperCAmelCase__      :					List[str]				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor" )
   # If remote code is disabled, we can't load this config.
   with self.assertRaises(snake_case__ ):
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )

   UpperCAmelCase__      :					Optional[Any]				       =							AutoFeatureExtractor.from_pretrained(
       "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
   self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )

   # Test feature extractor can be reloaded.
   with tempfile.TemporaryDirectory() as tmp_dir:
     feature_extractor.save_pretrained(snake_case__ )
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(snake_case__			,				trust_remote_code=snake_case__ )
   self.assertEqual(reloaded_feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )







 def 							UpperCamelCase  (					self		:  str ):



   '''simple docstring'''




   try:
     AutoConfig.register("custom"			,				snake_case__ )
     AutoFeatureExtractor.register(snake_case__			,				snake_case__ )
     # Trying to register something existing in the Transformers library will raise an error
     with self.assertRaises(snake_case__ ):
       AutoFeatureExtractor.register(snake_case__			,				snake_case__ )

     # Now that the config is registered, it can be used as any other config with the auto-API
     UpperCAmelCase__      :					str				       =							CustomFeatureExtractor.from_pretrained(snake_case__ )
     with tempfile.TemporaryDirectory() as tmp_dir:
       feature_extractor.save_pretrained(snake_case__ )
       UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(snake_case__ )
       self.assertIsInstance(snake_case__			,				snake_case__ )

   finally:
     if "custom" in CONFIG_MAPPING._extra_content:
       del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
       del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







 def 							UpperCamelCase  (					self		:  str ):



   '''simple docstring'''




   class 							UpperCAmelCase_							(    A				):

    '''simple docstring'''
    lowercase_					:    Optional[Any]				=							True

   try:
     AutoConfig.register("custom"			,				snake_case__ )
     AutoFeatureExtractor.register(snake_case__			,				snake_case__ )
     # If remote code is not set, the default is to use local
     UpperCAmelCase__      :					Any				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor" )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(feature_extractor.is_local )

     # If remote code is disabled, we load the local one.
     UpperCAmelCase__      :					Optional[int]				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(feature_extractor.is_local )

     # If remote is enabled, we load from the Hub
     UpperCAmelCase__      :					int				       =							AutoFeatureExtractor.from_pretrained(
         "hf-internal-testing/test_dynamic_feature_extractor"			,				trust_remote_code=snake_case__ )
     self.assertEqual(feature_extractor.__class__.__name__			,				"NewFeatureExtractor" )
     self.assertTrue(not hasattr(snake_case__			,				"is_local" ) )

   finally:
     if "custom" in CONFIG_MAPPING._extra_content:
       del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
       del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]



