


"""simple docstring"""


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class snake_case					(     _UpperCamelCase   , unittest.TestCase):
		__UpperCamelCase							=	PreTrainedTokenizerFast
		__UpperCamelCase							=	False
		__UpperCamelCase							=	True
		__UpperCamelCase							=	'tokenizer_file'







		def        a_       (    self					:				int						)				->  Optional[int]:





						'''simple docstring'''







						_A         =	False  # because we don't have pretrained_vocab_files_map
						super().setUp()
						_A         =	True

						_A         =	["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
						_A         =	"SaulLu/dummy-tokenizer-bytelevel-bpe"

						# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
						_A         =	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

						_A         =	PreTrainedTokenizerFast.from_pretrained(model_paths[0]						)
						tokenizer.save_pretrained(self.tmpdirname						)







		def        a_       (    self					:				Any						)				->  Tuple:





						'''simple docstring'''







						pass







		def        a_       (    self					:				Optional[Any]						)				->  Union[str, Any]:





						'''simple docstring'''







						pass







		def        a_       (    self					:				List[Any]						)				->  Dict:





						'''simple docstring'''







						pass







		def        a_       (    self					:				Optional[int]						)				->  Tuple:





						'''simple docstring'''







						pass







		def        a_       (    self					:				Dict						)				->  List[Any]:





						'''simple docstring'''







						_A         =	self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
														try:
																		_A         =	tempfile.mkdtemp()
																		_A         =	self.rust_tokenizer_class.from_pretrained(a__			,      **a__						)

																		tokenizer.save_pretrained(self.tmpdirname						)
																		super().test_training_new_tokenizer()
														finally:
																		# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																		# is restored
																		shutil.rmtree(self.tmpdirname						)
																		_A         =	tmpdirname_orig







		def        a_       (    self					:				Optional[int]						)				->  Dict:





						'''simple docstring'''







						_A         =	self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
														try:
																		_A         =	tempfile.mkdtemp()
																		_A         =	self.rust_tokenizer_class.from_pretrained(a__			,      **a__						)

																		tokenizer.save_pretrained(self.tmpdirname						)
																		super().test_training_new_tokenizer_with_special_tokens_change()
														finally:
																		# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																		# is restored
																		shutil.rmtree(self.tmpdirname						)
																		_A         =	tmpdirname_orig







		def        a_       (    self					:				Any						)				->  Optional[Any]:





						'''simple docstring'''







						_A         =	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name						)

						_A         =	("a" for _ in range(10_00						))
						_A         =	tokenizer.train_new_from_iterator(text_iterator=a__			,      length=10_00			,      vocab_size=50						)

						_A         =	new_tokenizer.encode("a🤗"						)
						self.assertEqual(a__			,      [64, 1_72, 2_53, 97, 2_45]						)





@require_tokenizers
class snake_case					(     unittest.TestCase):







		def        a_       (    self					:				List[str]						)				->  Optional[Any]:





						'''simple docstring'''







						_A         =	AutoTokenizer.from_pretrained("bert-base-cased"						)
						_A         =	json.loads(tokenizer._tokenizer.to_str()						)
						_A         =	len(a__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
										# Hack to save this in the tokenizer_config.json
										_A         =	["tokenizer.4.0.0.json"]
										tokenizer.save_pretrained(a__						)
										json.dump(a__			,      open(os.path.join(a__			,      "tokenizer.4.0.0.json"						)			,      "w"						)						)

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										_A         =	AutoTokenizer.from_pretrained(a__						)
										self.assertEqual(len(a__						)			,      len(a__						) + 1						)
										_A         =	json.loads(new_tokenizer._tokenizer.to_str()						)
										self.assertIn("huggingface"			,      json_tokenizer["model"]["vocab"]						)

										# Will need to be adjusted if we reach v42 and this test is still here.
										# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
										shutil.move(os.path.join(a__			,      "tokenizer.4.0.0.json"						)			,      os.path.join(a__			,      "tokenizer.42.0.0.json"						)						)
										_A         =	["tokenizer.42.0.0.json"]
										tokenizer.save_pretrained(a__						)
										_A         =	AutoTokenizer.from_pretrained(a__						)
										self.assertEqual(len(a__						)			,      len(a__						)						)
										_A         =	json.loads(new_tokenizer._tokenizer.to_str()						)
										self.assertNotIn("huggingface"			,      json_tokenizer["model"]["vocab"]						)







		def        a_       (    self					:				Tuple						)				->  str:





						'''simple docstring'''







						_A         =	"hf-internal-testing/test-two-tokenizers"

						# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
						_A         =	AutoTokenizer.from_pretrained(a__						)
						self.assertEqual(len(a__						)			,      2_89_97						)
						_A         =	json.loads(tokenizer._tokenizer.to_str()						)
						self.assertIn("huggingface"			,      json_tokenizer["model"]["vocab"]						)

						# Testing an older version by monkey-patching the version in the module it's used.
						import transformers as old_transformers

						_A         =	"3.0.0"
						_A         =	old_transformers.models.auto.AutoTokenizer.from_pretrained(a__						)
						self.assertEqual(len(a__						)			,      2_89_96						)
						_A         =	json.loads(old_tokenizer._tokenizer.to_str()						)
						self.assertNotIn("huggingface"			,      json_tokenizer["model"]["vocab"]						)





@require_tokenizers
class snake_case					(     unittest.TestCase):







		def        a_       (    self					:				Union[str, Any]						)				->  int:





						'''simple docstring'''







						_A         =	PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"						)
						_A         =	"The Matrix is a 1999 science fiction action film."

						with concurrent.futures.ThreadPoolExecutor() as executor:
										_A         =	[executor.submit(self.fetch			,      a__			,      a__						) for i in range(10						)]
										_A         =	[future.result() for future in futures]
										self.assertEqual(a__			,      [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10						)]						)







		def        a_       (    self					:				int			,      a__					:				str			,      a__					:				str						)				->  Dict:





						'''simple docstring'''







						return tokenizer.encode(a__			,      truncation="longest_first"			,      padding="longest"						)



"""simple docstring"""


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
class snake_case					(     _UpperCamelCase):
		__UpperCamelCase							=	'AutoFeatureExtractor'
		__UpperCamelCase							=	'Speech2Text2Tokenizer'







		def __init__(    self					:				int			,      a__					:				Tuple			,      a__					:				Optional[Any]						)				->  str:





						'''simple docstring'''







						super().__init__(a__			,      a__						)
						_A         =	self.feature_extractor
						_A         =	False







		def __call__(    self					:				Optional[int]			,      *a__					:				List[str]			,      **a__					:				Any						)				->  List[Any]:





						'''simple docstring'''







						if self._in_target_context_manager:
										return self.current_processor(*a__			,      **a__						)

						if "raw_speech" in kwargs:
										warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."						)
										_A         =	kwargs.pop("raw_speech"						)
						else:
										_A         =	kwargs.pop("audio"			,      a__						)
						_A         =	kwargs.pop("sampling_rate"			,      a__						)
						_A         =	kwargs.pop("text"			,      a__						)
						if len(a__						) > 0:
										_A         =	args[0]
										_A         =	args[1:]

						if audio is None and text is None:
										raise ValueError("You need to specify either an `audio` or `text` input to process."						)

						if audio is not None:
										_A         =	self.feature_extractor(a__			,      *a__			,      sampling_rate=a__			,      **a__						)
						if text is not None:
										_A         =	self.tokenizer(a__			,      **a__						)

						if text is None:
										return inputs
						elif audio is None:
										return encodings
						else:
										_A         =	encodings["input_ids"]
										return inputs







		def        a_       (    self					:				List[Any]			,      *a__					:				Tuple			,      **a__					:				List[Any]						)				->  List[Any]:





						'''simple docstring'''







						return self.tokenizer.batch_decode(*a__			,      **a__						)







		def        a_       (    self					:				Any			,      *a__					:				Dict			,      **a__					:				Tuple						)				->  Union[str, Any]:





						'''simple docstring'''







						return self.tokenizer.decode(*a__			,      **a__						)







		@contextmanager
		def        a_       (    self					:				Optional[int]						)				->  Optional[int]:





						'''simple docstring'''







						warnings.warn(
						    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
						    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
						    "your audio inputs, or in a separate call."						)
						_A         =	True
						_A         =	self.tokenizer
						yield
						_A         =	self.feature_extractor
						_A         =	False