



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case    :						str  =						get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__  (	UpperCamelCase					,							unittest.TestCase						):
					_A     :    Any           =						XGLMTokenizer
					_A     :    Optional[Any]           =						XGLMTokenizerFast
					_A     :    str           =						True
					_A     :    Optional[Any]           =						True







					def  _lowercase			(       self      :     Optional[Any]):
											super().setUp()

											# We have a SentencePiece fixture for testing
											A__	:							Union[str, Any]						    = XGLMTokenizer(_A			,		keep_accents=_A)
											tokenizer.save_pretrained(self.tmpdirname)







					def  _lowercase			(       self      :     str):
											A__	:							Union[str, Any]						    = "<pad>"
											A__	:							Dict						    = 1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)			,		_A)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)			,		_A)







					def  _lowercase			(       self      :     str):
											A__	:							List[str]						    = list(self.get_tokenizer().get_vocab().keys())

											self.assertEqual(vocab_keys[0]			,		"<s>")
											self.assertEqual(vocab_keys[1]			,		"<pad>")
											self.assertEqual(len(_A)			,		1008)







					def  _lowercase			(       self      :     List[Any]):
											self.assertEqual(self.get_tokenizer().vocab_size			,		1008)







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Optional[Any]						    = XGLMTokenizer(_A			,		keep_accents=_A)

											A__	:							List[Any]						    = tokenizer.tokenize("This is a test")
											self.assertListEqual(_A			,		["▁This", "▁is", "▁a", "▁t", "est"])

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(_A)			,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,		)

											A__	:							Any						    = tokenizer.tokenize("I was born in 92000, and this is falsé.")
											self.assertListEqual(
											    _A			,		[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "9",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "é",
											        ".",
											    ]			,		)
											A__	:							Dict						    = tokenizer.convert_tokens_to_ids(_A)
											self.assertListEqual(
											    _A			,		[
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
											    ]			,		)

											A__	:							str						    = tokenizer.convert_ids_to_tokens(_A)
											self.assertListEqual(
											    _A			,		[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "<unk>",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "<unk>",
											        ".",
											    ]			,		)







					@cached_property
					def  _lowercase			(       self      :     str):
											return XGLMTokenizer.from_pretrained("facebook/xglm-564M")







					def  _lowercase			(       self      :     Optional[int]):
											with tempfile.NamedTemporaryFile() as f:
																	shutil.copyfile(_A			,		f.name)
																	A__	:							Union[str, Any]						    = XGLMTokenizer(f.name			,		keep_accents=_A)
																	A__	:							List[Any]						    = pickle.dumps(_A)
											pickle.loads(_A)







					def  _lowercase			(       self      :     Optional[Any]):
											if not self.test_rust_tokenizer:
																	return

											A__	:							Optional[Any]						    = self.get_tokenizer()
											A__	:							Tuple						    = self.get_rust_tokenizer()

											A__	:							List[Any]						    = "I was born in 92000, and this is falsé."

											A__	:							Tuple						    = tokenizer.tokenize(_A)
											A__	:							int						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[str]						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							Tuple						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = self.get_rust_tokenizer()
											A__	:							Optional[Any]						    = tokenizer.encode(_A)
											A__	:							List[str]						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)







					@slow
					def  _lowercase			(       self      :     Tuple):
											A__	:							Dict						    = "Hello World!"
											A__	:							Optional[Any]						    = [2, 3_1227, 4447, 35]

											self.assertListEqual(_A			,		self.big_tokenizer.encode(_A))







					@slow
					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							Optional[Any]						    = (
											    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
											    " add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"
											)
											# fmt: off
											A__	:							Tuple						    = [2, 1018, 67, 11, 1988, 2617, 5631, 278, 11, 3407, 48, 7_1630, 2_8085, 4, 3234, 157, 13, 6, 5, 6, 4, 3526, 768, 15, 659, 57, 298, 3983, 864, 129, 21, 6, 5, 1_3675, 377, 652, 7580, 1_0341, 155, 2817, 422, 1666, 7, 1674, 53, 113, 20_2277, 1_7892, 33, 60, 87, 4, 3234, 157, 61, 2667, 5_2376, 19, 88, 23, 735]
											# fmt: on

											self.assertListEqual(_A			,		self.big_tokenizer.encode(_A))







					@slow
					def  _lowercase			(       self      :     Dict):
											# fmt: off
											A__	:							Union[str, Any]						    = {
											    "input_ids": [[2, 10_8825, 1163, 15, 8_8010, 473, 1_5898, 157, 1_3672, 1857, 312, 8, 23_8021, 1163, 53, 1_3672, 1857, 312, 8, 5_3283, 18_2396, 8, 1_8566, 16, 3_6733, 4101, 8, 230, 24_4017, 12_2553, 7, 15, 13_2597, 4, 293, 1_2511, 7610, 4, 3414, 13_2597, 9, 4, 3_2361, 362, 4, 734, 2_8512, 3_2569, 18, 4, 3_2361, 2_6096, 1_4982, 73, 1_8715, 2_1433, 23_5261, 15, 492, 1_2427, 16, 53, 1_8715, 2_1433, 6_5454, 15, 2_3659, 563, 16, 278, 597, 2843, 595, 7931, 18_2396, 6_4186, 22, 886, 595, 13_2981, 53, 2_5540, 3449, 4_3982, 3_9901, 5951, 878, 330, 4, 2_7694, 8_0269, 312, 53, 6517, 1_1780, 611, 2_0408, 5], [2, 6, 13_2597, 67, 4_2897, 33, 592, 8, 16_3729, 2_5540, 361, 13_6997, 10_9514, 17_3230, 7, 501, 60, 10_2913, 196, 5631, 235, 6_3243, 473, 6, 23_1757, 74, 5277, 7905, 53, 3095, 3_7317, 22, 454, 18_3874, 5], [2, 268, 3_1298, 4_6530, 6, 13_2935, 4_3831, 7, 597, 32, 24, 3688, 9865, 5]],
											    "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
											}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A			,		model_name="facebook/xglm-564M"			,		padding=_A			,		)




import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


snake_case    :						List[str]  =						{
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 1_2_8,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 5_0,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 1_0,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 1_0,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}




@is_staging_test
class 							lowerCAmelCase__  (	unittest.TestCase						):







					@classmethod
					def  _lowercase			(       cls      :     str):
											A__	:							int						    = TOKEN
											HfFolder.save_token(_A)







					@classmethod
					def  _lowercase			(       cls      :     Optional[Any]):
											try:
																	delete_repo(token=cls._token			,		repo_id="test-config")
											except HTTPError:
																	pass

											try:
																	delete_repo(token=cls._token			,		repo_id="valid_org/test-config-org")
											except HTTPError:
																	pass

											try:
																	delete_repo(token=cls._token			,		repo_id="test-dynamic-config")
											except HTTPError:
																	pass







					def  _lowercase			(       self      :     Dict):
											A__	:							List[str]						    = BertConfig(
											    vocab_size=99			,		hidden_size=32			,		num_hidden_layers=5			,		num_attention_heads=4			,		intermediate_size=37)
											config.push_to_hub("test-config"			,		use_auth_token=self._token)

											A__	:							Dict						    = BertConfig.from_pretrained(F'{USER}/test-config')
											for k, v in config.to_dict().items():
																	if k != "transformers_version":
																							self.assertEqual(_A			,		getattr(_A			,		_A))

        # Reset repo
											delete_repo(token=self._token			,		repo_id="test-config")

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(_A			,		repo_id="test-config"			,		push_to_hub=_A			,		use_auth_token=self._token)

											A__	:							Tuple						    = BertConfig.from_pretrained(F'{USER}/test-config')
											for k, v in config.to_dict().items():
																	if k != "transformers_version":
																							self.assertEqual(_A			,		getattr(_A			,		_A))







					def  _lowercase			(       self      :     List[Any]):
											A__	:							List[Any]						    = BertConfig(
											    vocab_size=99			,		hidden_size=32			,		num_hidden_layers=5			,		num_attention_heads=4			,		intermediate_size=37)
											config.push_to_hub("valid_org/test-config-org"			,		use_auth_token=self._token)

											A__	:							Union[str, Any]						    = BertConfig.from_pretrained("valid_org/test-config-org")
											for k, v in config.to_dict().items():
																	if k != "transformers_version":
																							self.assertEqual(_A			,		getattr(_A			,		_A))

        # Reset repo
											delete_repo(token=self._token			,		repo_id="valid_org/test-config-org")

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(
																	    _A			,		repo_id="valid_org/test-config-org"			,		push_to_hub=_A			,		use_auth_token=self._token)

											A__	:							List[str]						    = BertConfig.from_pretrained("valid_org/test-config-org")
											for k, v in config.to_dict().items():
																	if k != "transformers_version":
																							self.assertEqual(_A			,		getattr(_A			,		_A))







					def  _lowercase			(       self      :     Optional[int]):
											CustomConfig.register_for_auto_class()
											A__	:							Optional[Any]						    = CustomConfig(attribute=42)

											config.push_to_hub("test-dynamic-config"			,		use_auth_token=self._token)

											# This has added the proper auto_map field to the config
											self.assertDictEqual(config.auto_map			,		{"AutoConfig": "custom_configuration.CustomConfig"})

											A__	:							Optional[Any]						    = AutoConfig.from_pretrained(F'{USER}/test-dynamic-config'			,		trust_remote_code=_A)
											# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
											self.assertEqual(new_config.__class__.__name__			,		"CustomConfig")
											self.assertEqual(new_config.attribute			,		42)




class 							lowerCAmelCase__  (	unittest.TestCase						):







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							List[str]						    = GPTaConfig()

											# attempt to modify each of int/float/bool/str config records and verify they were updated
											A__	:							Optional[int]						    = c.n_embd + 1  # int
											A__	:							Tuple						    = c.resid_pdrop + 1.0  # float
											A__	:							Any						    = not c.scale_attn_weights  # bool
											A__	:							List[Any]						    = c.summary_type + "foo"  # str
											c.update_from_string(
											    F'n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}')
											self.assertEqual(_A			,		c.n_embd			,		"mismatch for key: n_embd")
											self.assertEqual(_A			,		c.resid_pdrop			,		"mismatch for key: resid_pdrop")
											self.assertEqual(_A			,		c.scale_attn_weights			,		"mismatch for key: scale_attn_weights")
											self.assertEqual(_A			,		c.summary_type			,		"mismatch for key: summary_type")







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Any						    = PretrainedConfig()
											A__	:							Optional[Any]						    = [key for key in base_config.__dict__ if key not in config_common_kwargs]
											# If this part of the test fails, you have arguments to addin config_common_kwargs above.
											self.assertListEqual(
											    _A			,		["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"])
											A__	:							Optional[Any]						    = [key for key, value in config_common_kwargs.items() if value == getattr(_A			,		_A)]
											if len(_A) > 0:
																	raise ValueError(
																	    "The following keys are set with the default values in"
																	    " `test_configuration_common.config_common_kwargs` pick another value for them:"
																	    F' {", ".join(_A)}.')







					def  _lowercase			(       self      :     List[str]):
											with self.assertRaises(_A):
																	# config is in subfolder, the following should not work without specifying the subfolder
																	A__	:							Any						    = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder")

											A__	:							str						    = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder"			,		subfolder="bert")

											self.assertIsNotNone(_A)







					def  _lowercase			(       self      :     List[str]):
											# A mock response for an HTTP head request to emulate server down
											A__	:							Tuple						    = mock.Mock()
											A__	:							Dict						    = 500
											A__	:							Union[str, Any]						    = {}
											A__	:							List[Any]						    = HTTPError
											A__	:							Optional[int]						    = {}

											# Download this model to make sure it's in the cache.
											A__	:							Any						    = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")

											# Under the mock environment we get a 500 error when trying to reach the model.
											with mock.patch("requests.Session.request"			,		return_value=_A) as mock_head:
																	A__	:							List[str]						    = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")
																	# This check we did call the fake head request
																	mock_head.assert_called()







					def  _lowercase			(       self      :     Optional[int]):
											# This test is for deprecated behavior and can be removed in v5
											A__	:							List[str]						    = BertConfig.from_pretrained(
											    "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json")







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Union[str, Any]						    = AutoConfig.from_pretrained("bert-base-cased")
											A__	:							str						    = ["config.4.0.0.json"]

											with tempfile.TemporaryDirectory() as tmp_dir:
																	configuration.save_pretrained(_A)
																	A__	:							Dict						    = 2
																	json.dump(configuration.to_dict()			,		open(os.path.join(_A			,		"config.4.0.0.json")			,		"w"))

																	# This should pick the new configuration file as the version of Transformers is > 4.0.0
																	A__	:							Any						    = AutoConfig.from_pretrained(_A)
																	self.assertEqual(new_configuration.hidden_size			,		2)

																	# Will need to be adjusted if we reach v42 and this test is still here.
																	# Should pick the old configuration file as the version of Transformers is < 4.42.0
																	A__	:							str						    = ["config.42.0.0.json"]
																	A__	:							Optional[int]						    = 768
																	configuration.save_pretrained(_A)
																	shutil.move(os.path.join(_A			,		"config.4.0.0.json")			,		os.path.join(_A			,		"config.42.0.0.json"))
																	A__	:							Optional[int]						    = AutoConfig.from_pretrained(_A)
																	self.assertEqual(new_configuration.hidden_size			,		768)







					def  _lowercase			(       self      :     Optional[int]):
											# This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
											A__	:							List[str]						    = "hf-internal-testing/test-two-configs"

											import transformers as new_transformers

											A__	:							int						    = "v4.0.0"
											A__    ,       A__	:							str						    = new_transformers.models.auto.AutoConfig.from_pretrained(
											    _A			,		return_unused_kwargs=_A)
											self.assertEqual(new_configuration.hidden_size			,		2)
											# This checks `_configuration_file` ia not kept in the kwargs by mistake.
											self.assertDictEqual(_A			,		{})

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											A__	:							List[Any]						    = "v3.0.0"
											A__	:							Optional[Any]						    = old_transformers.models.auto.AutoConfig.from_pretrained(_A)
											self.assertEqual(old_configuration.hidden_size			,		768)