


import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class        a	(_lowercase  ,       unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							Tuple  						=						GPTaTokenizer
							__UpperCAmelCase					:							int  						=						GPTaTokenizerFast
							__UpperCAmelCase					:							Dict  						=						True
							__UpperCAmelCase					:							Optional[int]  						=						{'''add_prefix_space''': True}
							__UpperCAmelCase					:							Tuple  						=						False

							def 					__snake_case      (							self :						Tuple       )			->					int:
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														__snake_case	:    Any											=     [
														    "l",
														    "o",
														    "w",
														    "e",
														    "r",
														    "s",
														    "t",
														    "i",
														    "d",
														    "n",
														    "\u0120",
														    "\u0120l",
														    "\u0120n",
														    "\u0120lo",
														    "\u0120low",
														    "er",
														    "\u0120lowest",
														    "\u0120newer",
														    "\u0120wider",
														    "<unk>",
														    "<|endoftext|>",
														]
														__snake_case	:    int											=     dict(zip(A_					,			range(len(A_       )       )       )       )
														__snake_case	:    int											=     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
														__snake_case	:    Dict											=     {"unk_token": "<unk>"}

														__snake_case	:    Tuple											=     os.path.join(self.tmpdirname					,			VOCAB_FILES_NAMES["vocab_file"]       )
														__snake_case	:    str											=     os.path.join(self.tmpdirname					,			VOCAB_FILES_NAMES["merges_file"]       )
														with open(self.vocab_file					,			"w"					,			encoding="utf-8"       ) as fp:
																					fp.write(json.dumps(A_       ) + "\n"       )
														with open(self.merges_file					,			"w"					,			encoding="utf-8"       ) as fp:
																					fp.write("\n".join(A_       )       )

							def 					__snake_case      (							self :						Optional[int]					,			**lowerCamelCase :						int       )			->					Dict:
														kwargs.update(self.special_tokens_map       )
														return GPTaTokenizer.from_pretrained(self.tmpdirname					,			**A_       )

							def 					__snake_case      (							self :						int					,			**lowerCamelCase :						Any       )			->					Optional[int]:
														kwargs.update(self.special_tokens_map       )
														return GPTaTokenizerFast.from_pretrained(self.tmpdirname					,			**A_       )

							def 					__snake_case      (							self :						List[Any]					,			lowerCamelCase :						int       )			->					int:
														__snake_case	:    Union[str, Any]											=     "lower newer"
														__snake_case	:    str											=     "lower newer"
														return input_text, output_text

							def 					__snake_case      (							self :						int       )			->					Optional[int]:
														__snake_case	:    Any											=     GPTaTokenizer(self.vocab_file					,			self.merges_file					,			**self.special_tokens_map       )
														__snake_case	:    int											=     "lower newer"
														__snake_case	:    int											=     ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
														__snake_case	:    Optional[int]											=     tokenizer.tokenize(A_					,			add_prefix_space=A_       )
														self.assertListEqual(A_					,			A_       )

														__snake_case	:    List[Any]											=     tokens + [tokenizer.unk_token]
														__snake_case	:    Union[str, Any]											=     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(A_       )					,			A_       )

							def 					__snake_case      (							self :						Any       )			->					Optional[int]:
														if not self.test_rust_tokenizer:
																					return

														__snake_case	:    Any											=     self.get_tokenizer()
														__snake_case	:    Any											=     self.get_rust_tokenizer(add_prefix_space=A_       )

														__snake_case	:    List[Any]											=     "lower newer"

														# Testing tokenization
														__snake_case	:    Union[str, Any]											=     tokenizer.tokenize(A_					,			add_prefix_space=A_       )
														__snake_case	:    Union[str, Any]											=     rust_tokenizer.tokenize(A_       )
														self.assertListEqual(A_					,			A_       )

														# Testing conversion to ids without special tokens
														__snake_case	:    Union[str, Any]											=     tokenizer.encode(A_					,			add_special_tokens=A_					,			add_prefix_space=A_       )
														__snake_case	:    str											=     rust_tokenizer.encode(A_					,			add_special_tokens=A_       )
														self.assertListEqual(A_					,			A_       )

														# Testing conversion to ids with special tokens
														__snake_case	:    Optional[Any]											=     self.get_rust_tokenizer(add_prefix_space=A_       )
														__snake_case	:    List[Any]											=     tokenizer.encode(A_					,			add_prefix_space=A_       )
														__snake_case	:    Tuple											=     rust_tokenizer.encode(A_       )
														self.assertListEqual(A_					,			A_       )

														# Testing the unknown token
														__snake_case	:    List[Any]											=     tokens + [rust_tokenizer.unk_token]
														__snake_case	:    Optional[int]											=     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A_       )					,			A_       )

							def 					__snake_case      (							self :						Dict					,			*lowerCamelCase :						List[Any]					,			**lowerCamelCase :						Optional[int]       )			->					Any:
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass

							def 					__snake_case      (							self :						Any					,			lowerCamelCase :						Optional[int]=15       )			->					int:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
																												__snake_case	:    Tuple											=     self.rust_tokenizer_class.from_pretrained(A_					,			**A_       )

																												# Simple input
																												__snake_case	:    Optional[int]											=     "This is a simple input"
																												__snake_case	:    str											=     ["This is a simple input 1", "This is a simple input 2"]
																												__snake_case	:    Optional[Any]											=     ("This is a simple input", "This is a pair")
																												__snake_case	:    str											=     [
																												    ("This is a simple input 1", "This is a simple input 2"),
																												    ("This is a simple pair 1", "This is a simple pair 2"),
																												]

																												# Simple input tests
																												self.assertRaises(A_					,			tokenizer_r.encode					,			A_					,			max_length=A_					,			padding="max_length"       )

																												# Simple input
																												self.assertRaises(A_					,			tokenizer_r.encode_plus					,			A_					,			max_length=A_					,			padding="max_length"       )

																												# Simple input
																												self.assertRaises(
																												    A_					,			tokenizer_r.batch_encode_plus					,			A_					,			max_length=A_					,			padding="max_length"					,			)

																												# Pair input
																												self.assertRaises(A_					,			tokenizer_r.encode					,			A_					,			max_length=A_					,			padding="max_length"       )

																												# Pair input
																												self.assertRaises(A_					,			tokenizer_r.encode_plus					,			A_					,			max_length=A_					,			padding="max_length"       )

																												# Pair input
																												self.assertRaises(
																												    A_					,			tokenizer_r.batch_encode_plus					,			A_					,			max_length=A_					,			padding="max_length"					,			)

							def 					__snake_case      (							self :						Any       )			->					Dict:
														__snake_case	:    int											=     GPTaTokenizer.from_pretrained(self.tmpdirname					,			pad_token="<pad>"       )

														# Simple input
														__snake_case	:    List[Any]											=     "This is a simple input"
														__snake_case	:    str											=     ["This is a simple input looooooooong", "This is a simple input"]
														__snake_case	:    str											=     ("This is a simple input", "This is a pair")
														__snake_case	:    List[Any]											=     [
														    ("This is a simple input loooooong", "This is a simple input"),
														    ("This is a simple pair loooooong", "This is a simple pair"),
														]

														__snake_case	:    Union[str, Any]											=     tokenizer.pad_token_id

														__snake_case	:    Optional[Any]											=     tokenizer(A_					,			padding="max_length"					,			max_length=30					,			return_tensors="np"       )
														__snake_case	:    Optional[Any]											=     tokenizer(A_					,			padding=A_					,			truncate=A_					,			return_tensors="np"       )
														__snake_case	:    List[str]											=     tokenizer(*A_					,			padding="max_length"					,			max_length=60					,			return_tensors="np"       )
														__snake_case	:    List[str]											=     tokenizer(A_					,			padding=A_					,			truncate=A_					,			return_tensors="np"       )

														# s
														# test single string max_length padding
														self.assertEqual(out_s["input_ids"].shape[-1]					,			30       )
														self.assertTrue(pad_token_id in out_s["input_ids"]       )
														self.assertTrue(0 in out_s["attention_mask"]       )

														# s2
														# test automatic padding
														self.assertEqual(out_sa["input_ids"].shape[-1]					,			33       )
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["input_ids"][0]       )
														self.assertFalse(0 in out_sa["attention_mask"][0]       )
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["input_ids"][1]       )
														self.assertTrue(0 in out_sa["attention_mask"][1]       )

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["input_ids"].shape[-1]					,			60       )
														self.assertTrue(pad_token_id in out_p["input_ids"]       )
														self.assertTrue(0 in out_p["attention_mask"]       )

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["input_ids"].shape[-1]					,			52       )
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["input_ids"][0]       )
														self.assertFalse(0 in out_pa["attention_mask"][0]       )
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["input_ids"][1]       )
														self.assertTrue(0 in out_pa["attention_mask"][1]       )

							def 					__snake_case      (							self :						Optional[Any]       )			->					List[str]:
														__snake_case	:    List[str]											=     "$$$"
														__snake_case	:    int											=     GPTaTokenizer.from_pretrained(self.tmpdirname					,			bos_token=A_					,			add_bos_token=A_       )

														__snake_case	:    Optional[Any]											=     "This is a simple input"
														__snake_case	:    int											=     ["This is a simple input 1", "This is a simple input 2"]

														__snake_case	:    Optional[Any]											=     tokenizer.bos_token_id

														__snake_case	:    Any											=     tokenizer(A_       )
														__snake_case	:    int											=     tokenizer(A_       )

														self.assertEqual(out_s.input_ids[0]					,			A_       )
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

														__snake_case	:    Tuple											=     tokenizer.decode(out_s.input_ids       )
														__snake_case	:    str											=     tokenizer.batch_decode(out_sa.input_ids       )

														self.assertEqual(decode_s.split()[0]					,			A_       )
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )

							def 					__snake_case      (							self :						Any       )			->					Dict:
														pass






							def 					__snake_case      (							self :						Tuple       )			->					Any:
														# TODO: change to self.get_tokenizers() when the fast version is implemented
														__snake_case	:    Dict											=     [self.get_tokenizer(do_lower_case=A_					,			add_bos_token=A_       )]
														for tokenizer in tokenizers:
																					with self.subTest(F'{tokenizer.__class__.__name__}'       ):
																												__snake_case	:    Tuple											=     "Encode this."
																												__snake_case	:    List[str]											=     "This one too please."
																												__snake_case	:    Optional[int]											=     tokenizer.encode(A_					,			add_special_tokens=A_       )
																												encoded_sequence += tokenizer.encode(A_					,			add_special_tokens=A_       )
																												__snake_case	:    int											=     tokenizer.encode_plus(
																												    A_					,			A_					,			add_special_tokens=A_					,			return_special_tokens_mask=A_					,			)
																												__snake_case	:    Any											=     encoded_sequence_dict["input_ids"]
																												__snake_case	:    str											=     encoded_sequence_dict["special_tokens_mask"]
																												self.assertEqual(len(A_       )					,			len(A_       )       )

																												__snake_case	:    Optional[int]											=     [
																												    (x if not special_tokens_mask[i] else None) for i, x in enumerate(A_       )
																												]
																												__snake_case	:    Tuple											=     [x for x in filtered_sequence if x is not None]
																												self.assertEqual(A_					,			A_       )

@require_tokenizers
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						Optional[int]       )			->					List[str]:
														# More context:
														# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
														# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
														# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

														__snake_case	:    Optional[Any]											=     AutoTokenizer.from_pretrained("facebook/opt-350m"					,			from_slow=A_       )
														__snake_case	:    Optional[int]											=     "A photo of a cat"

														__snake_case	:    Optional[Any]											=     tokenizer.encode(
														    A_					,			)
														self.assertEqual(A_					,			[2, 250, 1345, 9, 10, 4758]       )
														tokenizer.save_pretrained("test_opt"       )

														__snake_case	:    List[str]											=     AutoTokenizer.from_pretrained("./test_opt"       )
														__snake_case	:    List[Any]											=     tokenizer.encode(
														    A_					,			)
														self.assertEqual(A_					,			[2, 250, 1345, 9, 10, 4758]       )

							def 					__snake_case      (							self :						str       )			->					Optional[int]:
														__snake_case	:    int											=     AutoTokenizer.from_pretrained("facebook/opt-350m"					,			use_slow=A_       )
														__snake_case	:    Dict											=     "A photo of a cat"

														__snake_case	:    Optional[Any]											=     tokenizer.encode(
														    A_					,			)
														# Same as above
														self.assertEqual(A_					,			[2, 250, 1345, 9, 10, 4758]       )






							@unittest.skip("This test is failing because of a bug in the fast tokenizer"       )
							def 					__snake_case      (							self :						Optional[Any]       )			->					Optional[Any]:
														__snake_case	:    Optional[int]											=     AutoTokenizer.from_pretrained("facebook/opt-350m"					,			from_slow=A_       )

														__snake_case	:    List[Any]											=     "bos"
														__snake_case	:    str											=     tokenizer.get_vocab()["bos"]

														__snake_case	:    Dict											=     "A photo of a cat"
														__snake_case	:    str											=     tokenizer.encode(
														    A_					,			)
														# We changed the bos token
														self.assertEqual(A_					,			[31957, 250, 1345, 9, 10, 4758]       )
														tokenizer.save_pretrained("./tok"       )
														__snake_case	:    List[str]											=     AutoTokenizer.from_pretrained("./tok"       )
														self.assertTrue(tokenizer.is_fast       )
														__snake_case	:    List[Any]											=     tokenizer.encode(
														    A_					,			)
														self.assertEqual(A_					,			[31957, 250, 1345, 9, 10, 4758]       )





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case  :	Optional[Any]			      =       logging.get_logger(__name__)

_snake_case  :	str			      =       {
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}

class        a	(_lowerCAmelCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							List[Any]  						=						"xlnet"
							__UpperCAmelCase					:							Dict  						=						["mems"]
							__UpperCAmelCase					:							Tuple  						=						{
							    "n_token": "vocab_size",  # Backward compatibility
							    "hidden_size": "d_model",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}

							def __init__(							self :						List[Any]					,			lowerCamelCase :						Optional[int]=32000					,			lowerCamelCase :						str=1024					,			lowerCamelCase :						int=24					,			lowerCamelCase :						List[Any]=16					,			lowerCamelCase :						Any=4096					,			lowerCamelCase :						Optional[Any]="gelu"					,			lowerCamelCase :						Optional[Any]=True					,			lowerCamelCase :						Optional[int]="bi"					,			lowerCamelCase :						Union[str, Any]=0.02					,			lowerCamelCase :						Union[str, Any]=1E-12					,			lowerCamelCase :						Dict=0.1					,			lowerCamelCase :						int=512					,			lowerCamelCase :						str=None					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						str=False					,			lowerCamelCase :						Any=False					,			lowerCamelCase :						Optional[Any]=-1					,			lowerCamelCase :						List[Any]=False					,			lowerCamelCase :						Optional[int]="last"					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						Union[str, Any]="tanh"					,			lowerCamelCase :						int=0.1					,			lowerCamelCase :						Any=5					,			lowerCamelCase :						List[str]=5					,			lowerCamelCase :						List[Any]=5					,			lowerCamelCase :						Tuple=1					,			lowerCamelCase :						Dict=2					,			**lowerCamelCase :						List[str]					,			)			->					List[Any]:
														__snake_case	:    Optional[Any]											=     vocab_size
														__snake_case	:    Any											=     d_model
														__snake_case	:    int											=     n_layer
														__snake_case	:    Optional[int]											=     n_head
														if d_model % n_head != 0:
																					raise ValueError(F'\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'       )
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    F'`d_head` ({kwargs["d_head"]}) should be equal to `d_model // n_head` ({d_model // n_head})'       )
														__snake_case	:    Any											=     d_model // n_head
														__snake_case	:    Tuple											=     ff_activation
														__snake_case	:    int											=     d_inner
														__snake_case	:    Optional[Any]											=     untie_r
														__snake_case	:    Optional[Any]											=     attn_type

														__snake_case	:    List[str]											=     initializer_range
														__snake_case	:    Tuple											=     layer_norm_eps

														__snake_case	:    str											=     dropout
														__snake_case	:    Any											=     mem_len
														__snake_case	:    Tuple											=     reuse_len
														__snake_case	:    Union[str, Any]											=     bi_data
														__snake_case	:    List[Any]											=     clamp_len
														__snake_case	:    Optional[int]											=     same_length

														__snake_case	:    List[str]											=     summary_type
														__snake_case	:    Any											=     summary_use_proj
														__snake_case	:    Any											=     summary_activation
														__snake_case	:    Optional[int]											=     summary_last_dropout
														__snake_case	:    Dict											=     start_n_top
														__snake_case	:    Optional[Any]											=     end_n_top

														__snake_case	:    List[str]											=     bos_token_id
														__snake_case	:    Any											=     pad_token_id
														__snake_case	:    Optional[Any]											=     eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead."					,			lowerCamelCase					,			)
																					__snake_case	:    Any											=     kwargs["use_cache"]

														__snake_case	:    int											=     use_mems_eval
														__snake_case	:    Optional[int]											=     use_mems_train
														super().__init__(pad_token_id=lowerCamelCase					,			bos_token_id=lowerCamelCase					,			eos_token_id=lowerCamelCase					,			**lowerCamelCase       )

							@property
							def 					__snake_case      (							self :						Any       )			->					Optional[int]:
														logger.info(F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )
														return -1






							@max_position_embeddings.setter
							def 					__snake_case      (							self :						Union[str, Any]					,			lowerCamelCase :						Any       )			->					Optional[Any]:
														# Message copied from Transformer-XL documentation
														raise NotImplementedError(
														    F'The model {self.model_type} is one of the few models that has no sequence length limit.'       )

