'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_	:		Tuple			=     get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE   , unittest.TestCase	):







				'''simple docstring'''
				UpperCAmelCase__							=							ReformerTokenizer
				UpperCAmelCase__							=							ReformerTokenizerFast
				UpperCAmelCase__							=							True
				UpperCAmelCase__							=							False
				UpperCAmelCase__							=							True





				def  snake_case__				(						self					:						List[Any]					)							->Dict:


										'''simple docstring'''




										super().setUp()

										_UpperCamelCase			:			List[str]   			=						ReformerTokenizer(lowercase__       ,		keep_accents=lowercase__					)
										tokenizer.save_pretrained(self.tmpdirname					)





				def  snake_case__				(						self					:						Union[str, Any]					)							->Optional[int]:


										'''simple docstring'''




										_UpperCamelCase			:			List[str]   			=						"<s>"
										_UpperCamelCase			:			Tuple   			=						1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__					)       ,		lowercase__					)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__					)       ,		lowercase__					)





				def  snake_case__				(						self					:						Dict					)							->str:


										'''simple docstring'''




										_UpperCamelCase			:			Tuple   			=						list(self.get_tokenizer().get_vocab().keys()					)

										self.assertEqual(vocab_keys[0]       ,		"<unk>"					)
										self.assertEqual(vocab_keys[1]       ,		"<s>"					)
										self.assertEqual(vocab_keys[-1]       ,		"j"					)
										self.assertEqual(len(lowercase__					)       ,		1_000					)





				def  snake_case__				(						self					:						List[str]					)							->Optional[int]:


										'''simple docstring'''




										self.assertEqual(self.get_tokenizer().vocab_size       ,		1_000					)





				def  snake_case__				(						self					:						Tuple					)							->str:


										'''simple docstring'''




										if not self.test_rust_tokenizer:
																return

										_UpperCamelCase			:			List[Any]   			=						self.get_tokenizer()
										_UpperCamelCase			:			Any   			=						self.get_rust_tokenizer()

										_UpperCamelCase			:			Union[str, Any]   			=						"I was born in 92000, and this is falsé."

										_UpperCamelCase			:			Dict   			=						tokenizer.tokenize(lowercase__					)
										_UpperCamelCase			:			Dict   			=						rust_tokenizer.tokenize(lowercase__					)
										self.assertListEqual(lowercase__       ,		lowercase__					)

										_UpperCamelCase			:			Any   			=						tokenizer.encode(lowercase__       ,		add_special_tokens=lowercase__					)
										_UpperCamelCase			:			Dict   			=						rust_tokenizer.encode(lowercase__       ,		add_special_tokens=lowercase__					)
										self.assertListEqual(lowercase__       ,		lowercase__					)

										_UpperCamelCase			:			Union[str, Any]   			=						self.get_rust_tokenizer()
										_UpperCamelCase			:			int   			=						tokenizer.encode(lowercase__					)
										_UpperCamelCase			:			Dict   			=						rust_tokenizer.encode(lowercase__					)
										self.assertListEqual(lowercase__       ,		lowercase__					)





				def  snake_case__				(						self					:						Optional[Any]       ,		lowercase__					:						int=15					)							->int:


										'''simple docstring'''




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																						_UpperCamelCase			:			Union[str, Any]   			=						self.rust_tokenizer_class.from_pretrained(lowercase__       ,		**lowercase__					)

																						# Simple input
																						_UpperCamelCase			:			Optional[int]   			=						"This is a simple input"
																						_UpperCamelCase			:			Optional[int]   			=						["This is a simple input 1", "This is a simple input 2"]
																						_UpperCamelCase			:			Dict   			=						("This is a simple input", "This is a pair")
																						_UpperCamelCase			:			Optional[Any]   			=						[
																						    ("This is a simple input 1", "This is a simple input 2"),
																						    ("This is a simple pair 1", "This is a simple pair 2"),
																						]

																						# Simple input tests
																						self.assertRaises(lowercase__       ,		tokenizer_r.encode       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"					)

																						# Simple input
																						self.assertRaises(lowercase__       ,		tokenizer_r.encode_plus       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"					)

																						# Simple input
																						self.assertRaises(
																						    lowercase__       ,		tokenizer_r.batch_encode_plus       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"       ,		)

																						# Pair input
																						self.assertRaises(lowercase__       ,		tokenizer_r.encode       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"					)

																						# Pair input
																						self.assertRaises(lowercase__       ,		tokenizer_r.encode_plus       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"					)

																						# Pair input
																						self.assertRaises(
																						    lowercase__       ,		tokenizer_r.batch_encode_plus       ,		lowercase__       ,		max_length=lowercase__       ,		padding="max_length"       ,		)





				def  snake_case__				(						self					:						int					)							->Tuple:


										'''simple docstring'''




										pass





				def  snake_case__				(						self					:						List[Any]					)							->List[str]:


										'''simple docstring'''




										_UpperCamelCase			:			Union[str, Any]   			=						ReformerTokenizer(lowercase__       ,		keep_accents=lowercase__					)

										_UpperCamelCase			:			Any   			=						tokenizer.tokenize("This is a test"					)
										self.assertListEqual(lowercase__       ,		["▁This", "▁is", "▁a", "▁t", "est"]					)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowercase__					)       ,		[285, 46, 10, 170, 382]       ,		)

										_UpperCamelCase			:			Dict   			=						tokenizer.tokenize("I was born in 92000, and this is falsé."					)
										self.assertListEqual(
										    lowercase__       ,		[
										        SPIECE_UNDERLINE + "I",
										        SPIECE_UNDERLINE + "was",
										        SPIECE_UNDERLINE + "b",
										        "or",
										        "n",
										        SPIECE_UNDERLINE + "in",
										        SPIECE_UNDERLINE + "",
										        "9",
										        "2",
										        "0",
										        "0",
										        "0",
										        ",",
										        SPIECE_UNDERLINE + "and",
										        SPIECE_UNDERLINE + "this",
										        SPIECE_UNDERLINE + "is",
										        SPIECE_UNDERLINE + "f",
										        "al",
										        "s",
										        "é",
										        ".",
										    ]       ,		)
										_UpperCamelCase			:			Union[str, Any]   			=						tokenizer.convert_tokens_to_ids(lowercase__					)
										self.assertListEqual(
										    lowercase__       ,		[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,		)

										_UpperCamelCase			:			Optional[Any]   			=						tokenizer.convert_ids_to_tokens(lowercase__					)
										self.assertListEqual(
										    lowercase__       ,		[
										        SPIECE_UNDERLINE + "I",
										        SPIECE_UNDERLINE + "was",
										        SPIECE_UNDERLINE + "b",
										        "or",
										        "n",
										        SPIECE_UNDERLINE + "in",
										        SPIECE_UNDERLINE + "",
										        "<unk>",
										        "2",
										        "0",
										        "0",
										        "0",
										        ",",
										        SPIECE_UNDERLINE + "and",
										        SPIECE_UNDERLINE + "this",
										        SPIECE_UNDERLINE + "is",
										        SPIECE_UNDERLINE + "f",
										        "al",
										        "s",
										        "<unk>",
										        ".",
										    ]       ,		)





				@cached_property
				def  snake_case__				(						self					:						str					)							->List[str]:


										'''simple docstring'''




										return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"					)





				@slow
				def  snake_case__				(						self					:						List[Any]					)							->Optional[int]:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[Any]   			=						"Hello World!"
										_UpperCamelCase			:			Optional[Any]   			=						[126, 32, 262, 152, 38, 72, 287]

										self.assertListEqual(lowercase__       ,		self.big_tokenizer.encode(lowercase__					)					)





				@slow
				def  snake_case__				(						self					:						List[str]					)							->Union[str, Any]:


										'''simple docstring'''




										_UpperCamelCase			:			Union[str, Any]   			=						(
										    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
										    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
										)
										_UpperCamelCase			:			Any   			=						[
										    108,
										    265,
										    24,
										    111,
										    4,
										    258,
										    156,
										    35,
										    28,
										    275,
										    3,
										    259,
										    297,
										    260,
										    84,
										    4,
										    35,
										    110,
										    44,
										    8,
										    259,
										    91,
										    268,
										    21,
										    11,
										    209,
										    274,
										    109,
										    266,
										    277,
										    117,
										    86,
										    93,
										    315,
										    258,
										    278,
										    258,
										    277,
										    258,
										    0,
										    258,
										    288,
										    258,
										    319,
										    258,
										    0,
										    258,
										    0,
										    258,
										    0,
										    258,
										    0,
										    258,
										    287,
										    258,
										    315,
										    258,
										    289,
										    258,
										    278,
										    99,
										    269,
										    266,
										    262,
										    8,
										    259,
										    241,
										    4,
										    217,
										    230,
										    268,
										    266,
										    55,
										    168,
										    106,
										    75,
										    193,
										    266,
										    223,
										    27,
										    49,
										    26,
										    282,
										    25,
										    264,
										    299,
										    19,
										    26,
										    0,
										    258,
										    277,
										    117,
										    86,
										    93,
										    176,
										    183,
										    270,
										    11,
										    262,
										    42,
										    61,
										    265,
										]

										self.assertListEqual(lowercase__       ,		self.big_tokenizer.encode(lowercase__					)					)





				@require_torch
				@slow
				def  snake_case__				(						self					:						Union[str, Any]					)							->Any:


										'''simple docstring'''




										import torch

										from transformers import ReformerConfig, ReformerModel

										# Build sequence
										_UpperCamelCase			:			Union[str, Any]   			=						list(self.big_tokenizer.get_vocab().keys()					)[:10]
										_UpperCamelCase			:			Union[str, Any]   			=						" ".join(lowercase__					)
										_UpperCamelCase			:			Dict   			=						self.big_tokenizer.encode_plus(lowercase__       ,		return_tensors="pt"					)
										_UpperCamelCase			:			Dict   			=						self.big_tokenizer.batch_encode_plus([sequence, sequence]       ,		return_tensors="pt"					)

										_UpperCamelCase			:			Optional[Any]   			=						ReformerConfig()
										# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
										_UpperCamelCase			:			List[str]   			=						encoded_sequence["input_ids"].shape
										_UpperCamelCase			:			Optional[int]   			=						ReformerModel(lowercase__					)

										# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
										assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

										with torch.no_grad():
																model(**lowercase__					)
																model(**lowercase__					)







				@slow
				def  snake_case__				(						self					:						Tuple					)							->Tuple:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						{"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										# This tokenizer does not know some characters like ")".
										# That is the reason why we use very simple texts here.
										# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
										_UpperCamelCase			:			Tuple   			=						[
										    "This is a very simple sentence.",
										    "The quick brown fox jumps over the lazy dog.",
										]

										self.tokenizer_integration_test_util(
										    expected_encoding=lowercase__       ,		model_name="google/reformer-crime-and-punishment"       ,		revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"       ,		padding=lowercase__       ,		sequences=lowercase__       ,		)



'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_	:		Union[str, Any]			=     logging.get_logger(__name__)

lowerCAmelCase_	:		List[str]			=     {
    """hustvl/yolos-small""": """https://huggingface.co/hustvl/yolos-small/resolve/main/config.json""",
    # See all YOLOS models at https://huggingface.co/models?filter=yolos
}




class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''
				UpperCAmelCase__							=							'''yolos'''





				def __init__(						self					:						int       ,		lowercase__					:						List[str]=768       ,		lowercase__					:						Optional[Any]=12       ,		lowercase__					:						Union[str, Any]=12       ,		lowercase__					:						Any=3_072       ,		lowercase__					:						List[Any]="gelu"       ,		lowercase__					:						Dict=0.0       ,		lowercase__					:						Any=0.0       ,		lowercase__					:						Dict=0.0_2       ,		lowercase__					:						Tuple=1e-12       ,		lowercase__					:						str=[512, 864]       ,		lowercase__					:						Dict=16       ,		lowercase__					:						int=3       ,		lowercase__					:						Optional[Any]=True       ,		lowercase__					:						List[Any]=100       ,		lowercase__					:						str=True       ,		lowercase__					:						str=False       ,		lowercase__					:						List[str]=1       ,		lowercase__					:						Dict=5       ,		lowercase__					:						str=2       ,		lowercase__					:						Optional[int]=5       ,		lowercase__					:						Optional[int]=2       ,		lowercase__					:						Optional[Any]=0.1       ,		**lowercase__					:						Union[str, Any]       ,		)							->Tuple:


										'''simple docstring'''




										super().__init__(**lowercase__					)

										_UpperCamelCase			:			Optional[int]   			=						hidden_size
										_UpperCamelCase			:			str   			=						num_hidden_layers
										_UpperCamelCase			:			Optional[Any]   			=						num_attention_heads
										_UpperCamelCase			:			Optional[int]   			=						intermediate_size
										_UpperCamelCase			:			Union[str, Any]   			=						hidden_act
										_UpperCamelCase			:			Tuple   			=						hidden_dropout_prob
										_UpperCamelCase			:			Any   			=						attention_probs_dropout_prob
										_UpperCamelCase			:			Optional[int]   			=						initializer_range
										_UpperCamelCase			:			str   			=						layer_norm_eps
										_UpperCamelCase			:			Optional[int]   			=						image_size
										_UpperCamelCase			:			int   			=						patch_size
										_UpperCamelCase			:			Dict   			=						num_channels
										_UpperCamelCase			:			List[str]   			=						qkv_bias
										_UpperCamelCase			:			Dict   			=						num_detection_tokens
										_UpperCamelCase			:			int   			=						use_mid_position_embeddings
										_UpperCamelCase			:			int   			=						auxiliary_loss
										# Hungarian matcher
										_UpperCamelCase			:			Tuple   			=						class_cost
										_UpperCamelCase			:			str   			=						bbox_cost
										_UpperCamelCase			:			str   			=						giou_cost
										# Loss coefficients
										_UpperCamelCase			:			List[str]   			=						bbox_loss_coefficient
										_UpperCamelCase			:			str   			=						giou_loss_coefficient
										_UpperCamelCase			:			str   			=						eos_coefficient





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''
				UpperCAmelCase__							=							version.parse('''1.11'''	)





				@property
				def  snake_case__				(						self					:						Union[str, Any]					)							->Mapping[str, Mapping[int, str]]:


										'''simple docstring'''




										return OrderedDict(
										    [
										        ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
										    ]					)





				@property
				def  snake_case__				(						self					:						Any					)							->float:


										'''simple docstring'''




										return 1e-4







				@property
				def  snake_case__				(						self					:						List[str]					)							->int:


										'''simple docstring'''




										return 12


