




import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 		_lowerCAmelCase       (							__lowerCAmelCase			,    unittest.TestCase	):
			"""simple docstring"""
			lowerCAmelCase__           =ProphetNetTokenizer
			lowerCAmelCase__           =False
			def 					UpperCAmelCase     (					self       )		->							Optional[Any]:





				"""simple docstring"""







				super().setUp()

				snake_case__						:     str		=[
				    '''[UNK]''',
				    '''[CLS]''',
				    '''[SEP]''',
				    '''[PAD]''',
				    '''[MASK]''',
				    '''want''',
				    '''##want''',
				    '''##ed''',
				    '''wa''',
				    '''un''',
				    '''runn''',
				    '''##ing''',
				    ''',''',
				    '''low''',
				    '''lowest''',
				]
				snake_case__						:     Dict		=os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''vocab_file''']       )
				with open(self.vocab_file     ,		'''w'''     ,		encoding='''utf-8'''       ) as vocab_writer:
					vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )
			def 					UpperCAmelCase     (					self     ,		__SCREAMING_SNAKE_CASE       )		->							List[str]:





				"""simple docstring"""







				snake_case__						:     Tuple		='''UNwant\u00E9d,running'''
				snake_case__						:     Optional[Any]		='''unwanted, running'''
				return input_text, output_text
			def 					UpperCAmelCase     (					self       )		->							List[str]:





				"""simple docstring"""







				snake_case__						:     Union[str, Any]		=self.tokenizer_class(self.vocab_file       )

				snake_case__						:     int		=tokenizer.tokenize('''UNwant\u00E9d,running'''       )
				self.assertListEqual(lowerCamelCase_     ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       )     ,		[9, 6, 7, 12, 10, 11]       )
			def 					UpperCAmelCase     (					self       )		->							Optional[Any]:





				"""simple docstring"""







				snake_case__						:     Optional[int]		=BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )     ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )
			def 					UpperCAmelCase     (					self       )		->							Optional[Any]:





				"""simple docstring"""







				snake_case__						:     Dict		=BasicTokenizer(do_lower_case=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )     ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )     ,		['''hello''']       )
			def 					UpperCAmelCase     (					self       )		->							str:





				"""simple docstring"""







				snake_case__						:     Dict		=BasicTokenizer(do_lower_case=lowerCamelCase_     ,		strip_accents=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )     ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )     ,		['''h\u00E9llo''']       )
			def 					UpperCAmelCase     (					self       )		->							str:





				"""simple docstring"""







				snake_case__						:     int		=BasicTokenizer(do_lower_case=lowerCamelCase_     ,		strip_accents=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )     ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )     ,		['''hello''']       )
			def 					UpperCAmelCase     (					self       )		->							Tuple:





				"""simple docstring"""







				snake_case__						:     List[Any]		=BasicTokenizer(do_lower_case=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )     ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
				self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )     ,		['''hello''']       )
			def 					UpperCAmelCase     (					self       )		->							Optional[int]:





				"""simple docstring"""







				snake_case__						:     int		=BasicTokenizer(do_lower_case=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )     ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
			def 					UpperCAmelCase     (					self       )		->							Any:





				"""simple docstring"""







				snake_case__						:     Tuple		=BasicTokenizer(do_lower_case=lowerCamelCase_     ,		strip_accents=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )     ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
			def 					UpperCAmelCase     (					self       )		->							int:





				"""simple docstring"""







				snake_case__						:     List[str]		=BasicTokenizer(do_lower_case=lowerCamelCase_     ,		strip_accents=lowerCamelCase_       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )     ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
			def 					UpperCAmelCase     (					self       )		->							Tuple:





				"""simple docstring"""







				snake_case__						:     List[Any]		=BasicTokenizer(do_lower_case=lowerCamelCase_     ,		never_split=['''[UNK]''']       )

				self.assertListEqual(
				    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )     ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )
			def 					UpperCAmelCase     (					self       )		->							Union[str, Any]:





				"""simple docstring"""







				snake_case__						:     Any		=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

				snake_case__						:     int		={}
				for i, token in enumerate(lowerCamelCase_       ):
					snake_case__						:     Optional[Any]		=i
				snake_case__						:     Tuple		=WordpieceTokenizer(vocab=lowerCamelCase_     ,		unk_token='''[UNK]'''       )

				self.assertListEqual(tokenizer.tokenize(''''''       )     ,		[]       )

				self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )     ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

				self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )     ,		['''[UNK]''', '''runn''', '''##ing''']       )
			@require_torch
			def 					UpperCAmelCase     (					self       )		->							Tuple:





				"""simple docstring"""







				snake_case__						:     List[Any]		=self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''       )

				snake_case__						:     Tuple		=['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
				snake_case__						:     Optional[int]		=[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
				snake_case__						:     List[Any]		=tokenizer(lowerCamelCase_     ,		padding=lowerCamelCase_     ,		return_tensors='''pt'''       )
				self.assertIsInstance(lowerCamelCase_     ,		lowerCamelCase_       )
				snake_case__						:     Optional[Any]		=list(batch.input_ids.numpy()[0]       )
				self.assertListEqual(lowerCamelCase_     ,		lowerCamelCase_       )

				self.assertEqual((2, 9)     ,		batch.input_ids.shape       )
				self.assertEqual((2, 9)     ,		batch.attention_mask.shape       )
			def 					UpperCAmelCase     (					self       )		->							str:





				"""simple docstring"""







				self.assertTrue(_is_whitespace(''' '''       )       )
				self.assertTrue(_is_whitespace('''\t'''       )       )
				self.assertTrue(_is_whitespace('''\r'''       )       )
				self.assertTrue(_is_whitespace('''\n'''       )       )
				self.assertTrue(_is_whitespace('''\u00A0'''       )       )

				self.assertFalse(_is_whitespace('''A'''       )       )
				self.assertFalse(_is_whitespace('''-'''       )       )
			def 					UpperCAmelCase     (					self       )		->							Optional[int]:





				"""simple docstring"""







				self.assertTrue(_is_control('''\u0005'''       )       )

				self.assertFalse(_is_control('''A'''       )       )
				self.assertFalse(_is_control(''' '''       )       )
				self.assertFalse(_is_control('''\t'''       )       )
				self.assertFalse(_is_control('''\r'''       )       )
			def 					UpperCAmelCase     (					self       )		->							List[Any]:





				"""simple docstring"""







				self.assertTrue(_is_punctuation('''-'''       )       )
				self.assertTrue(_is_punctuation('''$'''       )       )
				self.assertTrue(_is_punctuation('''`'''       )       )
				self.assertTrue(_is_punctuation('''.'''       )       )

				self.assertFalse(_is_punctuation('''A'''       )       )
				self.assertFalse(_is_punctuation(''' '''       )       )


			@slow
			def 					UpperCAmelCase     (					self       )		->							Tuple:





				"""simple docstring"""







				snake_case__						:     Union[str, Any]		=self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''       )

				snake_case__						:     List[str]		=tokenizer.encode('''sequence builders'''     ,		add_special_tokens=lowerCamelCase_       )
				snake_case__						:     Optional[int]		=tokenizer.encode('''multi-sequence build'''     ,		add_special_tokens=lowerCamelCase_       )

				snake_case__						:     List[Any]		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase_       )
				snake_case__						:     Dict		=tokenizer.build_inputs_with_special_tokens(lowerCamelCase_     ,		lowerCamelCase_       )

				assert encoded_sentence == text + [102]
				assert encoded_pair == text + [102] + text_a + [102]





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures""")
_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_SCREAMING_SNAKE_CASE 		=      get_tests_dir("""fixtures/dummy-config.json""")
class SCREAMING_SNAKE_CASE_		(      unittest.TestCase		):







   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase			  =      0







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""facebook/wav2vec2-base-960h"""				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Optional[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    with tempfile.TemporaryDirectory() as tmpdirname:
     UpperCamelCase			  =      WavaVecaConfig()

     # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				).to_dict()

     config_dict.pop("""feature_extractor_type"""				)
     UpperCamelCase			  =      WavaVecaFeatureExtractor(**lowerCamelCase_				)

     # save in new folder
     model_config.save_pretrained(lowerCamelCase_				)
     config.save_pretrained(lowerCamelCase_				)

     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)

     # make sure private variable is not incorrectly saved
     UpperCamelCase			  =      json.loads(config.to_json_string()				)
     self.assertTrue("""_processor_class""" not in dict_as_saved				)

    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						List[str]				):





    """simple docstring"""


    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
    self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   """bert-base is not a local folder and is not a valid model identifier"""				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""bert-base"""				)







   def 						lowerCamelCase_	(     self						:						str				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_   ,   revision="""aaaaaa"""				)







   def 						lowerCamelCase_	(     self						:						Any				):





    """simple docstring"""


    with self.assertRaisesRegex(
        lowerCamelCase_   ,   """hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."""   ,   ):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained("""hf-internal-testing/config-no-model"""				)







   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    with self.assertRaises(lowerCamelCase_				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""				)
    # If remote code is disabled, we can't load this config.
    with self.assertRaises(lowerCamelCase_				):
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)

    UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
        """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
    self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)

    # Test feature extractor can be reloaded.
    with tempfile.TemporaryDirectory() as tmp_dir:
     feature_extractor.save_pretrained(lowerCamelCase_				)
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_   ,   trust_remote_code=lowerCamelCase_				)
    self.assertEqual(reloaded_feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)







   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    try:
     AutoConfig.register("""custom"""   ,   lowerCamelCase_				)
     AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)
     # Trying to register something existing in the Transformers library will raise an error
     with self.assertRaises(lowerCamelCase_				):
      AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)

     # Now that the config is registered, it can be used as any other config with the auto-API
     UpperCamelCase			  =      CustomFeatureExtractor.from_pretrained(lowerCamelCase_				)
     with tempfile.TemporaryDirectory() as tmp_dir:
      feature_extractor.save_pretrained(lowerCamelCase_				)
      UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(lowerCamelCase_				)
      self.assertIsInstance(lowerCamelCase_   ,   lowerCamelCase_				)

    finally:
     if "custom" in CONFIG_MAPPING._extra_content:
      del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
      del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







   def 						lowerCamelCase_	(     self						:						Dict				):





    """simple docstring"""


    class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):
       __lowerCAmelCase           =				True

    try:
     AutoConfig.register("""custom"""   ,   lowerCamelCase_				)
     AutoFeatureExtractor.register(lowerCamelCase_   ,   lowerCamelCase_				)
     # If remote code is not set, the default is to use local
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(feature_extractor.is_local				)

     # If remote code is disabled, we load the local one.
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(feature_extractor.is_local				)

     # If remote is enabled, we load from the Hub
     UpperCamelCase			  =      AutoFeatureExtractor.from_pretrained(
         """hf-internal-testing/test_dynamic_feature_extractor"""   ,   trust_remote_code=lowerCamelCase_				)
     self.assertEqual(feature_extractor.__class__.__name__   ,   """NewFeatureExtractor"""				)
     self.assertTrue(not hasattr(lowerCamelCase_   ,   """is_local"""				)				)

    finally:
     if "custom" in CONFIG_MAPPING._extra_content:
      del CONFIG_MAPPING._extra_content["custom"]
     if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
      del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
