
import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow





class 			_lowerCAmelCase      :




				def   __a		(self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   int:
							return None





class 			_lowerCAmelCase      :




				def   __a		(self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   Any:
							return None





class 			_lowerCAmelCase      (			unittest.TestCase ):
				_lowercase    	=[
				    # (model_name, model_kwargs)
				    ('''bert-base-cased''', {}),
				    ('''gpt2''', {'''use_cache''': False}),  # We don't support exporting GPT2 past keys anymore
				]




				@require_tf
				@slow
				def   __a		(self  )       ->   Tuple:
							for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
										self._test_export(_UpperCamelCase       ,       "tf"       ,       12       ,       **_UpperCamelCase  )




				@require_torch
				@slow
				def   __a		(self  )       ->   Optional[int]:
							for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
										self._test_export(_UpperCamelCase       ,       "pt"       ,       12       ,       **_UpperCamelCase  )




				@require_torch
				@slow
				def   __a		(self  )       ->   Tuple:
							from transformers import BertModel

							lowerCAmelCase_      			=			["[UNK]", "[SEP]", "[CLS]", "[PAD]", "[MASK]", "some", "other", "words"]
							with NamedTemporaryFile(mode="w+t"  ) as vocab_file:
										vocab_file.write("\n".join(_UpperCamelCase  )  )
										vocab_file.flush()
										lowerCAmelCase_      			=			BertTokenizerFast(vocab_file.name  )

							with TemporaryDirectory() as bert_save_dir:
										lowerCAmelCase_      			=			BertModel(BertConfig(vocab_size=len(_UpperCamelCase  )  )  )
										model.save_pretrained(_UpperCamelCase  )
										self._test_export(_UpperCamelCase       ,       "pt"       ,       12       ,       _UpperCamelCase  )




				@require_tf
				@slow
				def   __a		(self  )       ->   str:
							for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
										lowerCAmelCase_      			=			self._test_export(_UpperCamelCase       ,       "tf"       ,       12       ,       **_UpperCamelCase  )
										lowerCAmelCase_      			=			quantize(Path(_UpperCamelCase  )  )

										# Ensure the actual quantized model is not bigger than the original one
										if quantized_path.stat().st_size >= Path(_UpperCamelCase  ).stat().st_size:
													self.fail("Quantized model is bigger than initial ONNX model"  )




				@require_torch
				@slow
				def   __a		(self  )       ->   Any:
							for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
										lowerCAmelCase_      			=			self._test_export(_UpperCamelCase       ,       "pt"       ,       12       ,       **_UpperCamelCase  )
										lowerCAmelCase_      			=			quantize(_UpperCamelCase  )

										# Ensure the actual quantized model is not bigger than the original one
										if quantized_path.stat().st_size >= Path(_UpperCamelCase  ).stat().st_size:
													self.fail("Quantized model is bigger than initial ONNX model"  )




				def   __a		(self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase=None       ,       **_UpperCamelCase  )       ->   Any:
							try:
										# Compute path
										with TemporaryDirectory() as tempdir:
													lowerCAmelCase_      			=			Path(_UpperCamelCase  ).joinpath("model.onnx"  )

										# Remove folder if exists
										if path.parent.exists():
													path.parent.rmdir()

										# Export
										convert(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       **_UpperCamelCase  )

										return path
							except Exception as e:
										self.fail(_UpperCamelCase  )




				@require_torch
				@require_tokenizers
				@slow
				def   __a		(self  )       ->   Dict:
							from transformers import BertModel

							lowerCAmelCase_      			=			BertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random"  )  )
							lowerCAmelCase_      			=			BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random"  )
							self._test_infer_dynamic_axis(_UpperCamelCase       ,       _UpperCamelCase       ,       "pt"  )




				@require_tf
				@require_tokenizers
				@slow
				def   __a		(self  )       ->   Optional[Any]:
							from transformers import TFBertModel

							lowerCAmelCase_      			=			TFBertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random"  )  )
							lowerCAmelCase_      			=			BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random"  )
							self._test_infer_dynamic_axis(_UpperCamelCase       ,       _UpperCamelCase       ,       "tf"  )




				def   __a		(self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   Tuple:
							lowerCAmelCase_      			=			FeatureExtractionPipeline(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			["input_ids", "token_type_ids", "attention_mask", "output_0", "output_1"]
							lowerCAmelCase_			,						lowerCAmelCase_			,						lowerCAmelCase_			,						lowerCAmelCase_      			=			infer_shapes(_UpperCamelCase       ,       _UpperCamelCase  )

							# Assert all variables are present
							self.assertEqual(len(_UpperCamelCase  )       ,       len(_UpperCamelCase  )  )
							self.assertTrue(all(var_name in shapes for var_name in variable_names  )  )
							self.assertSequenceEqual(variable_names[:3]       ,       _UpperCamelCase  )
							self.assertSequenceEqual(variable_names[3:]       ,       _UpperCamelCase  )

							# Assert inputs are {0: batch, 1: sequence}
							for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
										self.assertDictEqual(shapes[var_name]       ,       {0: "batch", 1: "sequence"}  )

							# Assert outputs are {0: batch, 1: sequence} and {0: batch}
							self.assertDictEqual(shapes["output_0"]       ,       {0: "batch", 1: "sequence"}  )
							self.assertDictEqual(shapes["output_1"]       ,       {0: "batch"}  )




				def   __a		(self  )       ->   Dict:
							lowerCAmelCase_      			=			["input_ids", "attention_mask", "token_type_ids"]
							lowerCAmelCase_      			=			{"input_ids": [1, 2, 3, 4], "attention_mask": [0, 0, 0, 0], "token_type_ids": [1, 1, 1, 1]}
							lowerCAmelCase_			,						lowerCAmelCase_      			=			ensure_valid_input(FuncContiguousArgs()       ,       _UpperCamelCase       ,       _UpperCamelCase  )

							# Should have exactly the same number of args (all are valid)
							self.assertEqual(len(_UpperCamelCase  )       ,       3  )

							# Should have exactly the same input names
							self.assertEqual(set(_UpperCamelCase  )       ,       set(_UpperCamelCase  )  )

							# Parameter should be reordered according to their respective place in the function:
							# (input_ids, token_type_ids, attention_mask)
							self.assertEqual(_UpperCamelCase       ,       (tokens["input_ids"], tokens["token_type_ids"], tokens["attention_mask"])  )

							# Generated args are interleaved with another args (for instance parameter "past" in GPT2)
							lowerCAmelCase_			,						lowerCAmelCase_      			=			ensure_valid_input(FuncNonContiguousArgs()       ,       _UpperCamelCase       ,       _UpperCamelCase  )

							# Should have exactly the one arg (all before the one not provided "some_other_args")
							self.assertEqual(len(_UpperCamelCase  )       ,       1  )
							self.assertEqual(len(_UpperCamelCase  )       ,       1  )

							# Should have only "input_ids"
							self.assertEqual(inputs_args[0]       ,       tokens["input_ids"]  )
							self.assertEqual(ordered_input_names[0]       ,       "input_ids"  )




				def   __a		(self  )       ->   List[str]:
							lowerCAmelCase_      			=			generate_identified_filename(Path("/home/something/my_fake_model.onnx"  )       ,       "-test"  )
							self.assertEqual("/home/something/my_fake_model-test.onnx"       ,       generated.as_posix()  )





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   str:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				@require_tokenizers
				def   __a		(      self  )       ->   int:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				def   __a		(      self  )       ->   List[Any]:
							# This test is for deprecated behavior and can be removed in v5
							try:
										lowerCAmelCase_      			=			tempfile.mktemp()
										with open(_UpperCamelCase       ,       "wb"  ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       ,       _UpperCamelCase  )

										lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained(_UpperCamelCase  )
							finally:
										os.remove(_UpperCamelCase  )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("tokenizer.json"  ):
										# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
										return
							try:
										with open("tokenizer.json"       ,       "wb"  ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"       ,       _UpperCamelCase  )
										lowerCAmelCase_      			=			AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"  )
										# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
										self.assertEqual(tokenizer.vocab_size       ,       1_000  )
										# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
										os.remove("tokenizer.json"  )




				def   __a		(      self  )       ->   Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  )





@is_staging_test
class 			_lowerCAmelCase      (			unittest.TestCase ):
				_lowercase    	=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							lowerCAmelCase_      			=			TOKEN
							HfFolder.save_token(_UpperCamelCase  )




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							try:
										delete_repo(token=cls._token       ,       repo_id="test-tokenizer"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="valid_org/test-tokenizer-org"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="test-dynamic-tokenizer"  )
							except HTTPError:
										pass




				def   __a		(      self  )       ->   int:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("test-tokenizer"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="test-tokenizer"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_UpperCamelCase       ,       repo_id="test-tokenizer"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				def   __a		(      self  )       ->   Optional[int]:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("valid_org/test-tokenizer-org"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="valid_org/test-tokenizer-org"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(
										    _UpperCamelCase       ,       repo_id="valid_org/test-tokenizer-org"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				@require_tokenizers
				def   __a		(      self  )       ->   Optional[Any]:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			CustomTokenizer(_UpperCamelCase  )

							# No fast custom tokenizer
							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )

										lowerCAmelCase_      			=			BertTokenizerFast.from_pretrained(_UpperCamelCase  )
										bert_tokenizer.save_pretrained(_UpperCamelCase  )
										lowerCAmelCase_      			=			CustomTokenizerFast.from_pretrained(_UpperCamelCase  )

							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizerFast"  )
							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(
							    f"""{USER}/test-dynamic-tokenizer"""       ,       use_fast=_UpperCamelCase       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			Trie()
							trie.add("Hello 友達"  )
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}  )
							trie.add("Hello"  )
							trie.data
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}  )




				def   __a		(      self  )       ->   Optional[int]:
							lowerCAmelCase_      			=			Trie()
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS] This is a extra_id_100"]  )
							trie.add("[CLS]"  )
							trie.add("extra_id_1"  )
							trie.add("extra_id_100"  )
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS]", " This is a ", "extra_id_100"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["A", "BC"]  )
							self.assertEqual(trie.split("BCA"  )       ,       ["BC", "A"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("TOKEN]"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							trie.add("P"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("AB"  )
							trie.add("B"  )
							trie.add("C"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["AB", "C"]  )




				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			Trie()
							trie.add("ABC"  )
							trie.add("B"  )
							trie.add("CD"  )
							self.assertEqual(trie.split("ABCD"  )       ,       ["ABC", "D"]  )




				def   __a		(      self  )       ->   List[Any]:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							lowerCAmelCase_      			=			Trie()
							lowerCAmelCase_      			=			trie.cut_text("ABC"       ,       [0, 0, 2, 1, 2, 3]  )
							self.assertEqual(_UpperCamelCase       ,       ["AB", "C"]  )



