





'''simple docstring'''



import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


snake_case_ :	Optional[Any]             =   get_tests_dir('''fixtures/dummy_feature_extractor_config.json''')
snake_case_ :	Optional[Any]             =   get_tests_dir('''fixtures/vocab.json''')
snake_case_ :	Union[str, Any]             =   get_tests_dir('''fixtures''')

class      A_			(	unittest.TestCase	):

			'''simple docstring'''





			_lowerCAmelCase							=     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




			def    a						(       self ):
								_UpperCamelCase									= 0




			def    a						(       self ):
								_UpperCamelCase									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h" )
								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								with tempfile.TemporaryDirectory() as tmpdirname:
													_UpperCamelCase									= WavaVecaConfig()
													_UpperCamelCase									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h" )

													# save in new folder
													model_config.save_pretrained(A__ )
													processor.save_pretrained(A__ )

													_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )

								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								with tempfile.TemporaryDirectory() as tmpdirname:
													# copy relevant files
													copyfile(A__      ,  os.path.join(A__      ,  A__ ) )
													copyfile(A__      ,  os.path.join(A__      ,  "vocab.json" ) )

													_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )

								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								with tempfile.TemporaryDirectory() as tmpdirname:
													_UpperCamelCase									= WavaVecaFeatureExtractor()
													_UpperCamelCase									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h" )

													_UpperCamelCase									= WavaVecaProcessor(A__      ,  A__ )

													# save in new folder
													processor.save_pretrained(A__ )

													# drop `processor_class` in tokenizer
													with open(os.path.join(A__      ,  A__ )      ,  "r" ) as f:
																		_UpperCamelCase									= json.load(A__ )
																		config_dict.pop("processor_class" )

													with open(os.path.join(A__      ,  A__ )      ,  "w" ) as f:
																		f.write(json.dumps(A__ ) )

													_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )

								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								with tempfile.TemporaryDirectory() as tmpdirname:
													_UpperCamelCase									= WavaVecaFeatureExtractor()
													_UpperCamelCase									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h" )

													_UpperCamelCase									= WavaVecaProcessor(A__      ,  A__ )

													# save in new folder
													processor.save_pretrained(A__ )

													# drop `processor_class` in feature extractor
													with open(os.path.join(A__      ,  A__ )      ,  "r" ) as f:
																		_UpperCamelCase									= json.load(A__ )
																		config_dict.pop("processor_class" )

													with open(os.path.join(A__      ,  A__ )      ,  "w" ) as f:
																		f.write(json.dumps(A__ ) )

													_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )

								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								with tempfile.TemporaryDirectory() as tmpdirname:
													_UpperCamelCase									= WavaVecaConfig(processor_class="Wav2Vec2Processor" )
													model_config.save_pretrained(A__ )
													# copy relevant files
													copyfile(A__      ,  os.path.join(A__      ,  "vocab.json" ) )
													# create emtpy sample processor
													with open(os.path.join(A__      ,  A__ )      ,  "w" ) as f:
																		f.write("{}" )

													_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )

								self.assertIsInstance(A__      ,  A__ )




			def    a						(       self ):
								# If remote code is not set, we will time out when asking whether to load the model.
								with self.assertRaises(A__ ):
													_UpperCamelCase									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor" )
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(A__ ):
													_UpperCamelCase									= AutoProcessor.from_pretrained(
													    "hf-internal-testing/test_dynamic_processor"      ,  trust_remote_code=A__ )

								_UpperCamelCase									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"      ,  trust_remote_code=A__ )
								self.assertTrue(processor.special_attribute_present )
								self.assertEqual(processor.__class__.__name__      ,  "NewProcessor" )

								_UpperCamelCase									= processor.feature_extractor
								self.assertTrue(feature_extractor.special_attribute_present )
								self.assertEqual(feature_extractor.__class__.__name__      ,  "NewFeatureExtractor" )

								_UpperCamelCase									= processor.tokenizer
								self.assertTrue(tokenizer.special_attribute_present )
								if is_tokenizers_available():
													self.assertEqual(tokenizer.__class__.__name__      ,  "NewTokenizerFast" )

													# Test we can also load the slow version
													_UpperCamelCase									= AutoProcessor.from_pretrained(
													    "hf-internal-testing/test_dynamic_processor"      ,  trust_remote_code=A__      ,  use_fast=A__ )
													_UpperCamelCase									= new_processor.tokenizer
													self.assertTrue(new_tokenizer.special_attribute_present )
													self.assertEqual(new_tokenizer.__class__.__name__      ,  "NewTokenizer" )
								else:
													self.assertEqual(tokenizer.__class__.__name__      ,  "NewTokenizer" )




			def    a						(       self ):
								try:
													AutoConfig.register("custom"      ,  A__ )
													AutoFeatureExtractor.register(A__      ,  A__ )
													AutoTokenizer.register(A__      ,  slow_tokenizer_class=A__ )
													AutoProcessor.register(A__      ,  A__ )
													# Trying to register something existing in the Transformers library will raise an error
													with self.assertRaises(A__ ):
																		AutoProcessor.register(A__      ,  A__ )

													# Now that the config is registered, it can be used as any other config with the auto-API
													_UpperCamelCase									= CustomFeatureExtractor.from_pretrained(A__ )

													with tempfile.TemporaryDirectory() as tmp_dir:
																		_UpperCamelCase									= os.path.join(A__      ,  "vocab.txt" )
																		with open(A__      ,  "w"      ,  encoding="utf-8" ) as vocab_writer:
																							vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )
																		_UpperCamelCase									= CustomTokenizer(A__ )

													_UpperCamelCase									= CustomProcessor(A__      ,  A__ )

													with tempfile.TemporaryDirectory() as tmp_dir:
																		processor.save_pretrained(A__ )
																		_UpperCamelCase									= AutoProcessor.from_pretrained(A__ )
																		self.assertIsInstance(A__      ,  A__ )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																		del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]
													if CustomConfig in PROCESSOR_MAPPING._extra_content:
																		del PROCESSOR_MAPPING._extra_content[CustomConfig]




			def    a						(       self ):

								class      A_			(	__snake_case	):

											'''simple docstring'''





											_lowerCAmelCase							=     False




								class      A_			(	__snake_case	):

											'''simple docstring'''





											_lowerCAmelCase							=     False




								class      A_			(	__snake_case	):

											'''simple docstring'''





											_lowerCAmelCase							=     "AutoFeatureExtractor"
											_lowerCAmelCase							=     "AutoTokenizer"
											_lowerCAmelCase							=     False

								try:
													AutoConfig.register("custom"      ,  A__ )
													AutoFeatureExtractor.register(A__      ,  A__ )
													AutoTokenizer.register(A__      ,  slow_tokenizer_class=A__ )
													AutoProcessor.register(A__      ,  A__ )
													# If remote code is not set, the default is to use local classes.
													_UpperCamelCase									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor" )
													self.assertEqual(processor.__class__.__name__      ,  "NewProcessor" )
													self.assertFalse(processor.special_attribute_present )
													self.assertFalse(processor.feature_extractor.special_attribute_present )
													self.assertFalse(processor.tokenizer.special_attribute_present )

													# If remote code is disabled, we load the local ones.
													_UpperCamelCase									= AutoProcessor.from_pretrained(
													    "hf-internal-testing/test_dynamic_processor"      ,  trust_remote_code=A__ )
													self.assertEqual(processor.__class__.__name__      ,  "NewProcessor" )
													self.assertFalse(processor.special_attribute_present )
													self.assertFalse(processor.feature_extractor.special_attribute_present )
													self.assertFalse(processor.tokenizer.special_attribute_present )

													# If remote is enabled, we load from the Hub.
													_UpperCamelCase									= AutoProcessor.from_pretrained(
													    "hf-internal-testing/test_dynamic_processor"      ,  trust_remote_code=A__ )
													self.assertEqual(processor.__class__.__name__      ,  "NewProcessor" )
													self.assertTrue(processor.special_attribute_present )
													self.assertTrue(processor.feature_extractor.special_attribute_present )
													self.assertTrue(processor.tokenizer.special_attribute_present )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																		del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]
													if CustomConfig in PROCESSOR_MAPPING._extra_content:
																		del PROCESSOR_MAPPING._extra_content[CustomConfig]




			def    a						(       self ):
								_UpperCamelCase									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert" )
								self.assertEqual(processor.__class__.__name__      ,  "BertTokenizerFast" )




			def    a						(       self ):
								_UpperCamelCase									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext" )
								self.assertEqual(processor.__class__.__name__      ,  "ConvNextImageProcessor" )




@is_staging_test
class      A_			(	unittest.TestCase	):

			'''simple docstring'''





			_lowerCAmelCase							=     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




			@classmethod
			def    a						(       cls ):
								_UpperCamelCase									= TOKEN
								HfFolder.save_token(A__ )




			@classmethod
			def    a						(       cls ):
								try:
													delete_repo(token=cls._token      ,  repo_id="test-processor" )
								except HTTPError:
													pass

								try:
													delete_repo(token=cls._token      ,  repo_id="valid_org/test-processor-org" )
								except HTTPError:
													pass

								try:
													delete_repo(token=cls._token      ,  repo_id="test-dynamic-processor" )
								except HTTPError:
													pass




			def    a						(       self ):
								_UpperCamelCase									= WavaVecaProcessor.from_pretrained(A__ )
								with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(
													    os.path.join(A__      ,  "test-processor" )      ,  push_to_hub=A__      ,  use_auth_token=self._token )

													_UpperCamelCase									= WavaVecaProcessor.from_pretrained(F"{USER}/test-processor" )
													for k, v in processor.feature_extractor.__dict__.items():
																		self.assertEqual(A__      ,  getattr(new_processor.feature_extractor      ,  A__ ) )
													self.assertDictEqual(new_processor.tokenizer.get_vocab()      ,  processor.tokenizer.get_vocab() )




			def    a						(       self ):
								_UpperCamelCase									= WavaVecaProcessor.from_pretrained(A__ )

								with tempfile.TemporaryDirectory() as tmp_dir:
													processor.save_pretrained(
													    os.path.join(A__      ,  "test-processor-org" )      ,  push_to_hub=A__      ,  use_auth_token=self._token      ,  organization="valid_org"      ,  )

													_UpperCamelCase									= WavaVecaProcessor.from_pretrained("valid_org/test-processor-org" )
													for k, v in processor.feature_extractor.__dict__.items():
																		self.assertEqual(A__      ,  getattr(new_processor.feature_extractor      ,  A__ ) )
													self.assertDictEqual(new_processor.tokenizer.get_vocab()      ,  processor.tokenizer.get_vocab() )




			def    a						(       self ):
								CustomFeatureExtractor.register_for_auto_class()
								CustomTokenizer.register_for_auto_class()
								CustomProcessor.register_for_auto_class()

								_UpperCamelCase									= CustomFeatureExtractor.from_pretrained(A__ )

								with tempfile.TemporaryDirectory() as tmp_dir:
													_UpperCamelCase									= os.path.join(A__      ,  "vocab.txt" )
													with open(A__      ,  "w"      ,  encoding="utf-8" ) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens] ) )
													_UpperCamelCase									= CustomTokenizer(A__ )

								_UpperCamelCase									= CustomProcessor(A__      ,  A__ )

								with tempfile.TemporaryDirectory() as tmp_dir:
													create_repo(F"{USER}/test-dynamic-processor"      ,  token=self._token )
													_UpperCamelCase									= Repository(A__      ,  clone_from=F"{USER}/test-dynamic-processor"      ,  token=self._token )
													processor.save_pretrained(A__ )

													# This has added the proper auto_map field to the feature extractor config
													self.assertDictEqual(
													    processor.feature_extractor.auto_map      ,  {
													        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
													        "AutoProcessor": "custom_processing.CustomProcessor",
													    }      ,  )

													# This has added the proper auto_map field to the tokenizer config
													with open(os.path.join(A__      ,  "tokenizer_config.json" ) ) as f:
																		_UpperCamelCase									= json.load(A__ )
													self.assertDictEqual(
													    tokenizer_config["auto_map"]      ,  {
													        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
													        "AutoProcessor": "custom_processing.CustomProcessor",
													    }      ,  )

													# The code has been copied from fixtures
													self.assertTrue(os.path.isfile(os.path.join(A__      ,  "custom_feature_extraction.py" ) ) )
													self.assertTrue(os.path.isfile(os.path.join(A__      ,  "custom_tokenization.py" ) ) )
													self.assertTrue(os.path.isfile(os.path.join(A__      ,  "custom_processing.py" ) ) )

													repo.push_to_hub()

								_UpperCamelCase									= AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"      ,  trust_remote_code=A__ )
								# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
								self.assertEqual(new_processor.__class__.__name__      ,  "CustomProcessor" )











def  a__ (__lowercase							:str					,  __lowercase							:str   )		-> float:

				def get_matched_characters(__lowercase							:str					,  __lowercase							:str   ) -> str:
								_A	:				Union[str, Any]   =      []
								_A	:				Dict   =      min(len(_stra   )					,  len(_stra   )   ) // 2
								for i, l in enumerate(_stra   ):
												_A	:				Tuple   =      int(max(0					,  i - limit   )   )
												_A	:				Union[str, Any]   =      int(min(i + limit + 1					,  len(_stra   )   )   )
												if l in _stra[left:right]:
																matched.append(__lowercase   )
																_A	:				List[Any]   =      f"""{_stra[0:_stra.index(__lowercase   )]} {_stra[_stra.index(__lowercase   ) + 1:]}"""

								return "".join(__lowercase   )

				# matching characters
				_A	:				str   =      get_matched_characters(__lowercase					,  __lowercase   )
				_A	:				Optional[Any]   =      get_matched_characters(__lowercase					,  __lowercase   )
				_A	:				Any   =      len(__lowercase   )

				# transposition
				_A	:				str   =      (
				    len([(ca, ca) for ca, ca in zip(__lowercase					,  __lowercase   ) if ca != ca]   ) // 2
				)

				if not match_count:
								_A	:				Optional[int]   =      0.0
				else:
								_A	:				str   =      (
								    1
								    / 3
								    * (
								        match_count / len(__lowercase   )
								        + match_count / len(__lowercase   )
								        + (match_count - transpositions) / match_count
								    )
								)

				# common prefix up to 4 characters
				_A	:				Any   =      0
				for ca, ca in zip(stra[:4]					,  stra[:4]   ):
								if ca == ca:
												prefix_len += 1
								else:
												break

				return jaro + 0.1 * prefix_len * (1 - jaro)


if __name__ == "__main__":
	import doctest

	doctest.testmod()
	print(jaro_winkler('hello', 'world'))



