
"""simple docstring"""


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowerCamelCase__					(					__magic_name__ ,	unittest.TestCase				):





			'''simple docstring'''
			lowerCamelCase  					=		PreTrainedTokenizerFast
			lowerCamelCase  					=		False
			lowerCamelCase  					=		True
			lowerCamelCase  					=		'''tokenizer_file'''



			def 					_lowerCAmelCase		(							self  ) ->     Dict:
				_lowerCAmelCase								=False  # because we don't have pretrained_vocab_files_map
				super().setUp()
				_lowerCAmelCase								=True

				_lowerCAmelCase								=["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
				_lowerCAmelCase								="""SaulLu/dummy-tokenizer-bytelevel-bpe"""

				# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
				_lowerCAmelCase								=[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

				_lowerCAmelCase								=PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
				tokenizer.save_pretrained(self.tmpdirname  )



			def 					_lowerCAmelCase		(							self  ) ->     Optional[Any]:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     Any:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     List[str]:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     Optional[Any]:
				# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
				pass



			def 					_lowerCAmelCase		(							self  ) ->     str:
				_lowerCAmelCase								=self.tmpdirname
				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						try:
							_lowerCAmelCase								=tempfile.mkdtemp()
							_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase     ,     **__UpperCAmelCase  )

							tokenizer.save_pretrained(self.tmpdirname  )
							super().test_training_new_tokenizer()
						finally:
							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
							# is restored
							shutil.rmtree(self.tmpdirname  )
							_lowerCAmelCase								=tmpdirname_orig



			def 					_lowerCAmelCase		(							self  ) ->     List[str]:
				_lowerCAmelCase								=self.tmpdirname
				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						try:
							_lowerCAmelCase								=tempfile.mkdtemp()
							_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase     ,     **__UpperCAmelCase  )

							tokenizer.save_pretrained(self.tmpdirname  )
							super().test_training_new_tokenizer_with_special_tokens_change()
						finally:
							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
							# is restored
							shutil.rmtree(self.tmpdirname  )
							_lowerCAmelCase								=tmpdirname_orig



			def 					_lowerCAmelCase		(							self  ) ->     Tuple:
				_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

				_lowerCAmelCase								=("""a""" for _ in range(10_00  ))
				_lowerCAmelCase								=tokenizer.train_new_from_iterator(text_iterator=__UpperCAmelCase     ,     length=10_00     ,     vocab_size=50  )

				_lowerCAmelCase								=new_tokenizer.encode("""a🤗"""  )
				self.assertEqual(__UpperCAmelCase     ,     [64, 1_72, 2_53, 97, 2_45]  )







@require_tokenizers
class 		lowerCamelCase__					(					unittest.TestCase				):





			'''simple docstring'''



			def 					_lowerCAmelCase		(							self  ) ->     str:
				_lowerCAmelCase								=AutoTokenizer.from_pretrained("""bert-base-cased"""  )
				_lowerCAmelCase								=json.loads(tokenizer._tokenizer.to_str()  )
				_lowerCAmelCase								=len(__UpperCAmelCase  )

				with tempfile.TemporaryDirectory() as tmp_dir:
					# Hack to save this in the tokenizer_config.json
					_lowerCAmelCase								=["""tokenizer.4.0.0.json"""]
					tokenizer.save_pretrained(__UpperCAmelCase  )
					json.dump(__UpperCAmelCase     ,     open(os.path.join(__UpperCAmelCase     ,     """tokenizer.4.0.0.json"""  )     ,     """w"""  )  )

					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
					_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
					self.assertEqual(len(__UpperCAmelCase  )     ,     len(__UpperCAmelCase  ) + 1  )
					_lowerCAmelCase								=json.loads(new_tokenizer._tokenizer.to_str()  )
					self.assertIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )

					# Will need to be adjusted if we reach v42 and this test is still here.
					# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
					shutil.move(os.path.join(__UpperCAmelCase     ,     """tokenizer.4.0.0.json"""  )     ,     os.path.join(__UpperCAmelCase     ,     """tokenizer.42.0.0.json"""  )  )
					_lowerCAmelCase								=["""tokenizer.42.0.0.json"""]
					tokenizer.save_pretrained(__UpperCAmelCase  )
					_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
					self.assertEqual(len(__UpperCAmelCase  )     ,     len(__UpperCAmelCase  )  )
					_lowerCAmelCase								=json.loads(new_tokenizer._tokenizer.to_str()  )
					self.assertNotIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )



			def 					_lowerCAmelCase		(							self  ) ->     Any:
				# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
				_lowerCAmelCase								="""hf-internal-testing/test-two-tokenizers"""

				# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
				_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
				self.assertEqual(len(__UpperCAmelCase  )     ,     2_89_97  )
				_lowerCAmelCase								=json.loads(tokenizer._tokenizer.to_str()  )
				self.assertIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )

				# Testing an older version by monkey-patching the version in the module it's used.
				import transformers as old_transformers

				_lowerCAmelCase								="""3.0.0"""
				_lowerCAmelCase								=old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCAmelCase  )
				self.assertEqual(len(__UpperCAmelCase  )     ,     2_89_96  )
				_lowerCAmelCase								=json.loads(old_tokenizer._tokenizer.to_str()  )
				self.assertNotIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )







@require_tokenizers
class 		lowerCamelCase__					(					unittest.TestCase				):





			'''simple docstring'''



			def 					_lowerCAmelCase		(							self  ) ->     str:
				# See https://github.com/huggingface/transformers/pull/12550
				# and https://github.com/huggingface/tokenizers/issues/537
				_lowerCAmelCase								=PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""  )
				_lowerCAmelCase								="""The Matrix is a 1999 science fiction action film."""

				with concurrent.futures.ThreadPoolExecutor() as executor:
					_lowerCAmelCase								=[executor.submit(self.fetch     ,     __UpperCAmelCase     ,     __UpperCAmelCase  ) for i in range(10  )]
					_lowerCAmelCase								=[future.result() for future in futures]
					self.assertEqual(__UpperCAmelCase     ,     [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase  ) ->     Tuple:
				return tokenizer.encode(__UpperCAmelCase     ,     truncation="""longest_first"""     ,     padding="""longest"""  )





"""simple docstring"""


import enum
import inspect
import warnings
from functools import wraps
from typing import Callable, Optional

from .logging import get_logger


__A       						=       set()
__A       						=       get_logger(__name__)



def     _lowerCamelCase(__UpperCamelCase = None       )				-> Optional[Any]:

	def decorator(__UpperCamelCase       ):
		global _emitted_deprecation_warnings

		if inspect.isclass(__UpperCamelCase       ):
			_lowerCAmelCase								=deprecated_class_or_function.__init__
			_lowerCAmelCase								=deprecated_class_or_function.__name__
		else:
			_lowerCAmelCase								=deprecated_class_or_function
			_lowerCAmelCase								=deprecated_function.__name__
			# Support deprecating __init__ class method: class name instead
			_lowerCAmelCase								=name if name != """__init__""" else deprecated_function.__qualname__.split("""."""       )[-2]

		_lowerCAmelCase								=(
		    F'''{name} is deprecated and will be removed in the next major version of datasets.''' + F''' {help_message}'''
		    if help_message
		    else """"""
		)

		@wraps(__UpperCamelCase       )
		def wrapper(*__UpperCamelCase    , **__UpperCamelCase       ):
			_lowerCAmelCase								=hash(__UpperCamelCase       )
			if func_hash not in _emitted_deprecation_warnings:
				warnings.warn(__UpperCamelCase    , category=__UpperCamelCase    , stacklevel=2       )
				_emitted_deprecation_warnings.add(__UpperCamelCase       )
			return deprecated_function(*__UpperCamelCase    , **__UpperCamelCase       )

		_lowerCAmelCase								="""deprecated"""

		if inspect.isclass(__UpperCamelCase       ):
			_lowerCAmelCase								=wrapper
			return deprecated_class_or_function
		else:
			return wrapper

	return decorator







class 		lowerCamelCase__					(					enum.EnumMeta				):





			'''simple docstring'''



			def __getattribute__(							cls     ,     __UpperCAmelCase  ) ->     Optional[int]:
				_lowerCAmelCase								=super().__getattribute__(__UpperCAmelCase  )
				if isinstance(__UpperCAmelCase     ,     enum.Enum  ) and obj._on_access:
					obj._on_access()
				return obj



			def __getitem__(							cls     ,     __UpperCAmelCase  ) ->     List[str]:
				_lowerCAmelCase								=super().__getitem__(__UpperCAmelCase  )
				if member._on_access:
					member._on_access()
				return member



			def __call__(							cls     ,     __UpperCAmelCase     ,     __UpperCAmelCase=None     ,     *, __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=1  ) ->     Optional[Any]:
				_lowerCAmelCase								=super().__call__(__UpperCAmelCase     ,     __UpperCAmelCase     ,     module=__UpperCAmelCase     ,     qualname=__UpperCAmelCase     ,     type=__UpperCAmelCase     ,     start=__UpperCAmelCase  )
				if isinstance(__UpperCAmelCase     ,     enum.Enum  ) and obj._on_access:
					obj._on_access()
				return obj







class 		lowerCamelCase__					(					enum.Enum ,	metaclass=__magic_name__				):





			'''simple docstring'''



			def __new__(							cls     ,     __UpperCAmelCase  ) ->     Optional[int]:
				_lowerCAmelCase								=object.__new__(cls  )
				_lowerCAmelCase								=value
				_lowerCAmelCase								=member.deprecate
				return member



			@property
			def 					_lowerCAmelCase		(							self  ) ->     List[Any]:
				return ""



			def 					_lowerCAmelCase		(							self  ) ->     Tuple:
				_lowerCAmelCase								=f''' {self.help_message}''' if self.help_message else """"""
				warnings.warn(
				    f'''\'{self.__objclass__.__name__}\' is deprecated and will be removed in the next major version of datasets.'''
				    + help_message     ,     __UpperCAmelCase     ,     stacklevel=3     ,     )



