


import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_lowercase       :      Any	=	get_tests_dir("fixtures/dummy_feature_extractor_config.json")
_lowercase       :      Union[str, Any]	=	get_tests_dir("fixtures/vocab.json")
_lowercase       :      int	=	get_tests_dir("fixtures")






class       _UpperCamelCase    (					unittest.TestCase				):

				"""simple docstring"""







				lowerCAmelCase 		= ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']
				def      _UpperCAmelCase					(						self   )	->				List[Any]:
					A								=				0
				def      _UpperCAmelCase					(						self   )	->				Optional[Any]:
					A								=				AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""   )
					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				List[Any]:
					with tempfile.TemporaryDirectory() as tmpdirname:
						A								=				WavaVecaConfig()
						A								=				AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""   )

						# save in new folder
						model_config.save_pretrained(a__   )
						processor.save_pretrained(a__   )

						A								=				AutoProcessor.from_pretrained(a__   )

					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				Dict:
					with tempfile.TemporaryDirectory() as tmpdirname:
						# copy relevant files
						copyfile(a__				,   os.path.join(a__				,   a__   )   )
						copyfile(a__				,   os.path.join(a__				,   """vocab.json"""   )   )

						A								=				AutoProcessor.from_pretrained(a__   )

					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				int:
					with tempfile.TemporaryDirectory() as tmpdirname:
						A								=				WavaVecaFeatureExtractor()
						A								=				AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""   )

						A								=				WavaVecaProcessor(a__				,   a__   )

						# save in new folder
						processor.save_pretrained(a__   )

						# drop `processor_class` in tokenizer
						with open(os.path.join(a__				,   a__   )				,   """r"""   ) as f:
							A								=				json.load(a__   )
							config_dict.pop("""processor_class"""   )

						with open(os.path.join(a__				,   a__   )				,   """w"""   ) as f:
							f.write(json.dumps(a__   )   )

						A								=				AutoProcessor.from_pretrained(a__   )

					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				Optional[int]:
					with tempfile.TemporaryDirectory() as tmpdirname:
						A								=				WavaVecaFeatureExtractor()
						A								=				AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""   )

						A								=				WavaVecaProcessor(a__				,   a__   )

						# save in new folder
						processor.save_pretrained(a__   )

						# drop `processor_class` in feature extractor
						with open(os.path.join(a__				,   a__   )				,   """r"""   ) as f:
							A								=				json.load(a__   )
							config_dict.pop("""processor_class"""   )

						with open(os.path.join(a__				,   a__   )				,   """w"""   ) as f:
							f.write(json.dumps(a__   )   )

						A								=				AutoProcessor.from_pretrained(a__   )

					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				Tuple:
					with tempfile.TemporaryDirectory() as tmpdirname:
						A								=				WavaVecaConfig(processor_class="""Wav2Vec2Processor"""   )
						model_config.save_pretrained(a__   )
						# copy relevant files
						copyfile(a__				,   os.path.join(a__				,   """vocab.json"""   )   )
						# create emtpy sample processor
						with open(os.path.join(a__				,   a__   )				,   """w"""   ) as f:
							f.write("""{}"""   )

						A								=				AutoProcessor.from_pretrained(a__   )

					self.assertIsInstance(a__				,   a__   )
				def      _UpperCAmelCase					(						self   )	->				List[Any]:
					# If remote code is not set, we will time out when asking whether to load the model.
					with self.assertRaises(a__   ):
						A								=				AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""   )
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(a__   ):
						A								=				AutoProcessor.from_pretrained(
						    """hf-internal-testing/test_dynamic_processor"""				,   trust_remote_code=a__   )

					A								=				AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""				,   trust_remote_code=a__   )
					self.assertTrue(processor.special_attribute_present   )
					self.assertEqual(processor.__class__.__name__				,   """NewProcessor"""   )

					A								=				processor.feature_extractor
					self.assertTrue(feature_extractor.special_attribute_present   )
					self.assertEqual(feature_extractor.__class__.__name__				,   """NewFeatureExtractor"""   )

					A								=				processor.tokenizer
					self.assertTrue(tokenizer.special_attribute_present   )
					if is_tokenizers_available():
						self.assertEqual(tokenizer.__class__.__name__				,   """NewTokenizerFast"""   )

						# Test we can also load the slow version
						A								=				AutoProcessor.from_pretrained(
						    """hf-internal-testing/test_dynamic_processor"""				,   trust_remote_code=a__				,   use_fast=a__   )
						A								=				new_processor.tokenizer
						self.assertTrue(new_tokenizer.special_attribute_present   )
						self.assertEqual(new_tokenizer.__class__.__name__				,   """NewTokenizer"""   )
					else:
						self.assertEqual(tokenizer.__class__.__name__				,   """NewTokenizer"""   )
				def      _UpperCAmelCase					(						self   )	->				Optional[int]:
					try:
						AutoConfig.register("""custom"""				,   a__   )
						AutoFeatureExtractor.register(a__				,   a__   )
						AutoTokenizer.register(a__				,   slow_tokenizer_class=a__   )
						AutoProcessor.register(a__				,   a__   )
						# Trying to register something existing in the Transformers library will raise an error
						with self.assertRaises(a__   ):
							AutoProcessor.register(a__				,   a__   )

						# Now that the config is registered, it can be used as any other config with the auto-API
						A								=				CustomFeatureExtractor.from_pretrained(a__   )

						with tempfile.TemporaryDirectory() as tmp_dir:
							A								=				os.path.join(a__				,   """vocab.txt"""   )
							with open(a__				,   """w"""				,   encoding="""utf-8"""   ) as vocab_writer:
								vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]   )   )
							A								=				CustomTokenizer(a__   )

						A								=				CustomProcessor(a__				,   a__   )

						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(a__   )
							A								=				AutoProcessor.from_pretrained(a__   )
							self.assertIsInstance(a__				,   a__   )

					finally:
						if "custom" in CONFIG_MAPPING._extra_content:
							del CONFIG_MAPPING._extra_content["custom"]
						if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
							del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
						if CustomConfig in TOKENIZER_MAPPING._extra_content:
							del TOKENIZER_MAPPING._extra_content[CustomConfig]
						if CustomConfig in PROCESSOR_MAPPING._extra_content:
							del PROCESSOR_MAPPING._extra_content[CustomConfig]
				def      _UpperCAmelCase					(						self   )	->				Any:






					class       _UpperCamelCase    (					__snake_case				):

									"""simple docstring"""







									lowerCAmelCase 		= False






					class       _UpperCamelCase    (					__snake_case				):

									"""simple docstring"""







									lowerCAmelCase 		= False






					class       _UpperCamelCase    (					__snake_case				):

									"""simple docstring"""







									lowerCAmelCase 		= 'AutoFeatureExtractor'
									lowerCAmelCase 		= 'AutoTokenizer'
									lowerCAmelCase 		= False

					try:
						AutoConfig.register("""custom"""				,   a__   )
						AutoFeatureExtractor.register(a__				,   a__   )
						AutoTokenizer.register(a__				,   slow_tokenizer_class=a__   )
						AutoProcessor.register(a__				,   a__   )
						# If remote code is not set, the default is to use local classes.
						A								=				AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""   )
						self.assertEqual(processor.__class__.__name__				,   """NewProcessor"""   )
						self.assertFalse(processor.special_attribute_present   )
						self.assertFalse(processor.feature_extractor.special_attribute_present   )
						self.assertFalse(processor.tokenizer.special_attribute_present   )

						# If remote code is disabled, we load the local ones.
						A								=				AutoProcessor.from_pretrained(
						    """hf-internal-testing/test_dynamic_processor"""				,   trust_remote_code=a__   )
						self.assertEqual(processor.__class__.__name__				,   """NewProcessor"""   )
						self.assertFalse(processor.special_attribute_present   )
						self.assertFalse(processor.feature_extractor.special_attribute_present   )
						self.assertFalse(processor.tokenizer.special_attribute_present   )

						# If remote is enabled, we load from the Hub.
						A								=				AutoProcessor.from_pretrained(
						    """hf-internal-testing/test_dynamic_processor"""				,   trust_remote_code=a__   )
						self.assertEqual(processor.__class__.__name__				,   """NewProcessor"""   )
						self.assertTrue(processor.special_attribute_present   )
						self.assertTrue(processor.feature_extractor.special_attribute_present   )
						self.assertTrue(processor.tokenizer.special_attribute_present   )

					finally:
						if "custom" in CONFIG_MAPPING._extra_content:
							del CONFIG_MAPPING._extra_content["custom"]
						if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
							del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
						if CustomConfig in TOKENIZER_MAPPING._extra_content:
							del TOKENIZER_MAPPING._extra_content[CustomConfig]
						if CustomConfig in PROCESSOR_MAPPING._extra_content:
							del PROCESSOR_MAPPING._extra_content[CustomConfig]
				def      _UpperCAmelCase					(						self   )	->				Dict:
					A								=				AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )
					self.assertEqual(processor.__class__.__name__				,   """BertTokenizerFast"""   )

				def      _UpperCAmelCase					(						self   )	->				str:
					A								=				AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-convnext"""   )
					self.assertEqual(processor.__class__.__name__				,   """ConvNextImageProcessor"""   )






@is_staging_test
class       _UpperCamelCase    (					unittest.TestCase				):

				"""simple docstring"""







				lowerCAmelCase 		= ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']
				@classmethod
				def      _UpperCAmelCase					(						cls   )	->				str:
					A								=				TOKEN
					HfFolder.save_token(a__   )
				@classmethod
				def      _UpperCAmelCase					(						cls   )	->				Optional[int]:
					try:
						delete_repo(token=cls._token				,   repo_id="""test-processor"""   )
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token				,   repo_id="""valid_org/test-processor-org"""   )
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token				,   repo_id="""test-dynamic-processor"""   )
					except HTTPError:
						pass
				def      _UpperCAmelCase					(						self   )	->				Any:
					A								=				WavaVecaProcessor.from_pretrained(a__   )
					with tempfile.TemporaryDirectory() as tmp_dir:
						processor.save_pretrained(
						    os.path.join(a__				,   """test-processor"""   )				,   push_to_hub=a__				,   use_auth_token=self._token   )

						A								=				WavaVecaProcessor.from_pretrained(f'{USER}/test-processor'   )
						for k, v in processor.feature_extractor.__dict__.items():
							self.assertEqual(a__				,   getattr(new_processor.feature_extractor				,   a__   )   )
						self.assertDictEqual(new_processor.tokenizer.get_vocab()				,   processor.tokenizer.get_vocab()   )
				def      _UpperCAmelCase					(						self   )	->				Optional[int]:
					A								=				WavaVecaProcessor.from_pretrained(a__   )

					with tempfile.TemporaryDirectory() as tmp_dir:
						processor.save_pretrained(
						    os.path.join(a__				,   """test-processor-org"""   )				,   push_to_hub=a__				,   use_auth_token=self._token				,   organization="""valid_org"""				,   )

						A								=				WavaVecaProcessor.from_pretrained("""valid_org/test-processor-org"""   )
						for k, v in processor.feature_extractor.__dict__.items():
							self.assertEqual(a__				,   getattr(new_processor.feature_extractor				,   a__   )   )
						self.assertDictEqual(new_processor.tokenizer.get_vocab()				,   processor.tokenizer.get_vocab()   )

				def      _UpperCAmelCase					(						self   )	->				int:
					CustomFeatureExtractor.register_for_auto_class()
					CustomTokenizer.register_for_auto_class()
					CustomProcessor.register_for_auto_class()

					A								=				CustomFeatureExtractor.from_pretrained(a__   )

					with tempfile.TemporaryDirectory() as tmp_dir:
						A								=				os.path.join(a__				,   """vocab.txt"""   )
						with open(a__				,   """w"""				,   encoding="""utf-8"""   ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]   )   )
						A								=				CustomTokenizer(a__   )

					A								=				CustomProcessor(a__				,   a__   )

					with tempfile.TemporaryDirectory() as tmp_dir:
						create_repo(f'{USER}/test-dynamic-processor'				,   token=self._token   )
						A								=				Repository(a__				,   clone_from=f'{USER}/test-dynamic-processor'				,   token=self._token   )
						processor.save_pretrained(a__   )

						# This has added the proper auto_map field to the feature extractor config
						self.assertDictEqual(
						    processor.feature_extractor.auto_map				,   {
						        """AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor""",
						        """AutoProcessor""": """custom_processing.CustomProcessor""",
						    }				,   )

						# This has added the proper auto_map field to the tokenizer config
						with open(os.path.join(a__				,   """tokenizer_config.json"""   )   ) as f:
							A								=				json.load(a__   )
						self.assertDictEqual(
						    tokenizer_config["""auto_map"""]				,   {
						        """AutoTokenizer""": ["""custom_tokenization.CustomTokenizer""", None],
						        """AutoProcessor""": """custom_processing.CustomProcessor""",
						    }				,   )

						# The code has been copied from fixtures
						self.assertTrue(os.path.isfile(os.path.join(a__				,   """custom_feature_extraction.py"""   )   )   )
						self.assertTrue(os.path.isfile(os.path.join(a__				,   """custom_tokenization.py"""   )   )   )
						self.assertTrue(os.path.isfile(os.path.join(a__				,   """custom_processing.py"""   )   )   )

						repo.push_to_hub()

					A								=				AutoProcessor.from_pretrained(f'{USER}/test-dynamic-processor'				,   trust_remote_code=a__   )
					# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
					self.assertEqual(new_processor.__class__.__name__				,   """CustomProcessor"""   )







import math





def   _lowerCAmelCase				(   UpperCamelCase__:     int						) ->   bool:





	"""simple docstring"""







	if 1 < number < 4:
		# 2 and 3 are primes
		return True
	elif number < 2 or number % 2 == 0 or number % 3 == 0:
		# Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
		return False

	# All primes number are in format of 6k +/- 1
	for i in range(5							,					int(math.sqrt(UpperCamelCase__						) + 1						)							,					6						):
		if number % i == 0 or number % (i + 2) == 0:
			return False
	return True





def   _lowerCAmelCase				(   UpperCamelCase__:     float = 0.1						) ->   int:





	"""simple docstring"""







	A								=				3
	A								=				3

	while primes / (2 * j - 1) >= ratio:
		for i in range(j * j + j + 1							,					(j + 2) * (j + 2)							,					j + 1						):
			primes += is_prime(UpperCamelCase__						)
		j += 2
	return j


if __name__ == "__main__":
				import doctest

				doctest.testmod()



