



import os
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken
from ...utils import logging


SCREAMING_SNAKE_CASE__   			=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   			=   {'''vocab_file''': '''vocab.txt'''}

SCREAMING_SNAKE_CASE__   			=   {
    '''vocab_file''': {
        '''facebook/esm2_t6_8M_UR50D''': '''https://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txt''',
        '''facebook/esm2_t12_35M_UR50D''': '''https://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt''',
    },
}

SCREAMING_SNAKE_CASE__   			=   {
    '''facebook/esm2_t6_8M_UR50D''': 1_0_2_4,
    '''facebook/esm2_t12_35M_UR50D''': 1_0_2_4,
}


def A    (							__UpperCamelCase       )    ->   Optional[Any]:
			with open(__UpperCamelCase			,						'r'       ) as f:
						A__	  =     f.read().splitlines()
						return [l.strip() for l in lines]






class   __lowerCAmelCase  (		UpperCAmelCase_   ):
					"""simple docstring"""






					A__					:		List[str]      							=     VOCAB_FILES_NAMES
					A__					:		Dict      							=     PRETRAINED_VOCAB_FILES_MAP
					A__					:		Tuple      							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__					:		Optional[int]      							=     ["input_ids", "attention_mask"]



					def __init__(  self       :							Optional[Any]					,       _snake_case       :							Union[str, Any]					,       _snake_case       :							Tuple="<unk>"					,       _snake_case       :							Union[str, Any]="<cls>"					,       _snake_case       :							List[str]="<pad>"					,       _snake_case       :							Optional[Any]="<mask>"					,       _snake_case       :							Optional[Any]="<eos>"					,       **_snake_case       :							List[Any]					,       ):






								"""simple docstring"""


								super().__init__(**_snake_case       )
								A__	  =     load_vocab_file(_snake_case       )
								A__	  =     dict(enumerate(self.all_tokens       )       )
								A__	  =     {tok: ind for ind, tok in enumerate(self.all_tokens       )}
								A__	  =     unk_token
								A__	  =     cls_token
								A__	  =     pad_token
								A__	  =     mask_token
								A__	  =     eos_token
								A__	  =     self.all_tokens
								self._create_trie(self.unique_no_split_tokens       )



					def      _a				(  self       :							Optional[Any]					,       _snake_case       :							int       ):






								"""simple docstring"""


								return self._id_to_token.get(_snake_case					,       self.unk_token       )



					def      _a				(  self       :							List[Any]					,       _snake_case       :							str       ):






								"""simple docstring"""


								return self._token_to_id.get(_snake_case					,       self._token_to_id.get(self.unk_token       )       )



					def      _a				(  self       :							str					,       _snake_case       :							List[Any]					,       **_snake_case       :							Optional[Any]       ):






								"""simple docstring"""


								return text.split()



					def      _a				(  self       :							Any					,       _snake_case       :							str=False       ):






								"""simple docstring"""


								return len(self._id_to_token       )



					def      _a				(  self       :							Union[str, Any]       ):






								"""simple docstring"""


								return {token: i for i, token in enumerate(self.all_tokens       )}



					def      _a				(  self       :							Optional[Any]					,       _snake_case       :							str       ):






								"""simple docstring"""


								return self._token_to_id.get(_snake_case					,       self._token_to_id.get(self.unk_token       )       )



					def      _a				(  self       :							Union[str, Any]					,       _snake_case       :							int       ):






								"""simple docstring"""


								return self._id_to_token.get(_snake_case					,       self.unk_token       )



					def      _a				(  self       :							Tuple					,       _snake_case       :							List[int]					,       _snake_case       :							Optional[List[int]] = None       ):






								"""simple docstring"""


								A__	  =     [self.cls_token_id]
								A__	  =     [self.eos_token_id]  # No sep token in ESM vocabulary
								if token_ids_a is None:
											if self.eos_token_id is None:
														return cls + token_ids_a
											else:
														return cls + token_ids_a + sep
								elif self.eos_token_id is None:
											raise ValueError('Cannot tokenize multiple sequences when EOS token is not set!'       )
								return cls + token_ids_a + sep + token_ids_a + sep  # Multiple inputs always have an EOS token



					def      _a				(  self       :							List[Any]					,       _snake_case       :							List					,       _snake_case       :							Optional[List] = None					,       _snake_case       :							bool = False       ):






								"""simple docstring"""


								if already_has_special_tokens:
											if token_ids_a is not None:
														raise ValueError(
														    'You should not supply a second sequence if the provided sequence of '
														    'ids is already formatted with special tokens for the model.'       )

											return [1 if token in self.all_special_ids else 0 for token in token_ids_a]
								A__	  =     [1] + ([0] * len(_snake_case       )) + [1]
								if token_ids_a is not None:
											mask += [0] * len(_snake_case       ) + [1]
								return mask



					def      _a				(  self       :							Optional[int]					,       _snake_case       :							Union[str, Any]					,       _snake_case       :							int       ):






								"""simple docstring"""


								A__	  =     os.path.join(_snake_case					,       (filename_prefix + '-' if filename_prefix else '') + 'vocab.txt'       )
								with open(_snake_case					,       'w'       ) as f:
											f.write('\n'.join(self.all_tokens       )       )
								return (vocab_file,)



					@property
					def      _a				(  self       :							Union[str, Any]       ):






								"""simple docstring"""


								return self.get_vocab_size(with_added_tokens=_snake_case       )



					def      _a				(  self       :							Tuple					,       _snake_case       :							Union[List[str], List[AddedToken]]					,       _snake_case       :							bool = False       ):






								"""simple docstring"""


								return super()._add_tokens(_snake_case					,       special_tokens=_snake_case       )





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures""")
_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures/dummy-config.json""")
class SCREAMING_SNAKE_CASE_		(      unittest.TestCase		):







   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase			  =      0







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""facebook/wav2vec2-base-960h"""				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Optional[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    with tempfile.TemporaryDirectory() as tmpdirname:
     UpperCamelCase			  =      WavaVecaConfig()

     # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				).to_dict()

     config_dict.pop("""feature_extractor_type"""				)
     UpperCamelCase			  =      WavaVecaFeatureExtractor(**lowerCamelCase_				)

     # save in new folder
     model_config.save_pretrained(lowerCamelCase_				)
     config.save_pretrained(lowerCamelCase_				)

     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)

     # make sure private variable is not incorrectly saved
     UpperCamelCase			  =      json.loads(config.to_json_string()				)
     self.assertTrue("""_processor_class""" not in dict_as_saved				)

    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						List[str]				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   """bert-base is not a local folder and is not a valid model identifier"""				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""bert-base"""				)







   def 						lowerCamelCase_	(     self						:						str				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_   ,   revision="""aaaaaa"""				)







   def 						lowerCamelCase_	(     self						:						Any				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   """hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."""   ,   ):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""hf-internal-testing/config-no-model"""				)







   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    with self.assertRaises(lowerCamelCase_				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""				)
    # If remote code is disabled, we can't load this config.
    with self.assertRaises(lowerCamelCase_				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)

    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
        """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
    self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)

    # Test feature extractor can be reloaded.
    with tempfile.TemporaryDirectory() as tmp_dir:
     feature_extractor.save_pretrained(lowerCamelCase_				)
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_   ,   trust_remote_code=lowerCamelCase_				)
    self.assertEqual(reloaded_feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)







   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    try:
     AutoConfig.register("""custom"""   ,   lowerCamelCase_				)
     AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)
     # Trying to register something existing in the Transformers library will raise an error
     with self.assertRaises(lowerCamelCase_				):
      AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)

     # Now that the config is registered, it can be used as any other config with the auto-API
     UpperCamelCase			  =      CustomFeatureExtractor.from_pretrained(lowerCamelCase_				)
     with tempfile.TemporaryDirectory() as tmp_dir:
      feature_extractor.save_pretrained(lowerCamelCase_				)
      UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
      self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)

    finally:
     if "custom" in CONFIG_MAPPING._extra_content:
      del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
      del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







   def 						lowerCamelCase_	(     self						:						Dict				):





    """simple docstring"""


    class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):
       __lowerCAmelCase           =				True

    try:
     AutoConfig.register("""custom"""   ,   lowerCamelCase_				)
     AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)
     # If remote code is not set, the default is to use local
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(feature_extractor.is_local				)

     # If remote code is disabled, we load the local one.
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(feature_extractor.is_local				)

     # If remote is enabled, we load from the Hub
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(not hasattr(lowerCamelCase_   ,   """is_local"""				)				)

    finally:
     if "custom" in CONFIG_MAPPING._extra_content:
      del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
      del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
