




'''simple docstring'''
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

_SCREAMING_SNAKE_CASE			=						_symbol_database.Default()


_SCREAMING_SNAKE_CASE			=						_descriptor_pool.Default().AddSerializedFile(
    B"\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"
)

_SCREAMING_SNAKE_CASE			=						globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "sentencepiece_model_pb2", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
							_SCREAMING_SNAKE_CASE			=						None
							_SCREAMING_SNAKE_CASE			=						B"H\003"
							# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
							# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
							# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
							# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
							# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
							_SCREAMING_SNAKE_CASE			=						45
							_SCREAMING_SNAKE_CASE			=						15_81
							_SCREAMING_SNAKE_CASE			=						15_17
							_SCREAMING_SNAKE_CASE			=						15_70
							_SCREAMING_SNAKE_CASE			=						15_84
							_SCREAMING_SNAKE_CASE			=						17_93
							_SCREAMING_SNAKE_CASE			=						17_95
							_SCREAMING_SNAKE_CASE			=						19_16
							_SCREAMING_SNAKE_CASE			=						18_64
							_SCREAMING_SNAKE_CASE			=						19_05
							_SCREAMING_SNAKE_CASE			=						19_19
							_SCREAMING_SNAKE_CASE			=						24_29
							_SCREAMING_SNAKE_CASE			=						22_08
							_SCREAMING_SNAKE_CASE			=						24_18
							_SCREAMING_SNAKE_CASE			=						23_23
							_SCREAMING_SNAKE_CASE			=						24_07
# @@protoc_insertion_point(module_scope)










'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE			=						get_tests_dir("fixtures/test_sentencepiece.model")
@require_sentencepiece
@require_tokenizers
class   __UpperCAmelCase				(     lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		ReformerTokenizer
					_UpperCamelCase							=		ReformerTokenizerFast
					_UpperCamelCase							=		True
					_UpperCamelCase							=		False
					_UpperCamelCase							=		True



					def     __snake_case						(	self     :						Optional[int])    ->							int:
								super().setUp()

								A_									= ReformerTokenizer(_lowercase      ,       keep_accents=_lowercase)
								tokenizer.save_pretrained(self.tmpdirname)



					def     __snake_case						(	self     :						List[Any])    ->							str:
								A_									= '<s>'
								A_									= 1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase)      ,       _lowercase)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase)      ,       _lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[Any]:
								A_									= list(self.get_tokenizer().get_vocab().keys())

								self.assertEqual(vocab_keys[0]      ,       '<unk>')
								self.assertEqual(vocab_keys[1]      ,       '<s>')
								self.assertEqual(vocab_keys[-1]      ,       'j')
								self.assertEqual(len(_lowercase)      ,       1_000)



					def     __snake_case						(	self     :						int)    ->							Optional[int]:
								self.assertEqual(self.get_tokenizer().vocab_size      ,       1_000)



					def     __snake_case						(	self     :						List[Any])    ->							Any:
								if not self.test_rust_tokenizer:
											return

								A_									= self.get_tokenizer()
								A_									= self.get_rust_tokenizer()

								A_									= 'I was born in 92000, and this is falsé.'

								A_									= tokenizer.tokenize(_lowercase)
								A_									= rust_tokenizer.tokenize(_lowercase)
								self.assertListEqual(_lowercase      ,       _lowercase)

								A_									= tokenizer.encode(_lowercase      ,       add_special_tokens=_lowercase)
								A_									= rust_tokenizer.encode(_lowercase      ,       add_special_tokens=_lowercase)
								self.assertListEqual(_lowercase      ,       _lowercase)

								A_									= self.get_rust_tokenizer()
								A_									= tokenizer.encode(_lowercase)
								A_									= rust_tokenizer.encode(_lowercase)
								self.assertListEqual(_lowercase      ,       _lowercase)



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						Optional[int]=15)    ->							str:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
														A_									= self.rust_tokenizer_class.from_pretrained(_lowercase      ,       **_lowercase)

														# Simple input
														A_									= 'This is a simple input'
														A_									= ['This is a simple input 1', 'This is a simple input 2']
														A_									= ('This is a simple input', 'This is a pair')
														A_									= [
														    ('This is a simple input 1', 'This is a simple input 2'),
														    ('This is a simple pair 1', 'This is a simple pair 2'),
														]

														# Simple input tests
														self.assertRaises(_lowercase      ,       tokenizer_r.encode      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length')

														# Simple input
														self.assertRaises(_lowercase      ,       tokenizer_r.encode_plus      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length')

														# Simple input
														self.assertRaises(
														    _lowercase      ,       tokenizer_r.batch_encode_plus      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length'      ,       )

														# Pair input
														self.assertRaises(_lowercase      ,       tokenizer_r.encode      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length')

														# Pair input
														self.assertRaises(_lowercase      ,       tokenizer_r.encode_plus      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length')

														# Pair input
														self.assertRaises(
														    _lowercase      ,       tokenizer_r.batch_encode_plus      ,       _lowercase      ,       max_length=_lowercase      ,       padding='max_length'      ,       )



					def     __snake_case						(	self     :						Optional[int])    ->							Optional[int]:
								pass



					def     __snake_case						(	self     :						Tuple)    ->							Optional[Any]:
								A_									= ReformerTokenizer(_lowercase      ,       keep_accents=_lowercase)

								A_									= tokenizer.tokenize('This is a test')
								self.assertListEqual(_lowercase      ,       ['▁This', '▁is', '▁a', '▁t', 'est'])

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_lowercase)      ,       [285, 46, 10, 170, 382]      ,       )

								A_									= tokenizer.tokenize('I was born in 92000, and this is falsé.')
								self.assertListEqual(
								    _lowercase      ,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]      ,       )
								A_									= tokenizer.convert_tokens_to_ids(_lowercase)
								self.assertListEqual(
								    _lowercase      ,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,       )

								A_									= tokenizer.convert_ids_to_tokens(_lowercase)
								self.assertListEqual(
								    _lowercase      ,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]      ,       )



					@cached_property
					def     __snake_case						(	self     :						str)    ->							Any:
								return ReformerTokenizer.from_pretrained('google/reformer-crime-and-punishment')



					@slow
					def     __snake_case						(	self     :						Tuple)    ->							Union[str, Any]:
								A_									= 'Hello World!'
								A_									= [126, 32, 262, 152, 38, 72, 287]

								self.assertListEqual(_lowercase      ,       self.big_tokenizer.encode(_lowercase))



					@slow
					def     __snake_case						(	self     :						Optional[int])    ->							int:
								A_									= (
								    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
								    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
								)
								A_									= [
								    108,
								    265,
								    24,
								    111,
								    4,
								    258,
								    156,
								    35,
								    28,
								    275,
								    3,
								    259,
								    297,
								    260,
								    84,
								    4,
								    35,
								    110,
								    44,
								    8,
								    259,
								    91,
								    268,
								    21,
								    11,
								    209,
								    274,
								    109,
								    266,
								    277,
								    117,
								    86,
								    93,
								    315,
								    258,
								    278,
								    258,
								    277,
								    258,
								    0,
								    258,
								    288,
								    258,
								    319,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    287,
								    258,
								    315,
								    258,
								    289,
								    258,
								    278,
								    99,
								    269,
								    266,
								    262,
								    8,
								    259,
								    241,
								    4,
								    217,
								    230,
								    268,
								    266,
								    55,
								    168,
								    106,
								    75,
								    193,
								    266,
								    223,
								    27,
								    49,
								    26,
								    282,
								    25,
								    264,
								    299,
								    19,
								    26,
								    0,
								    258,
								    277,
								    117,
								    86,
								    93,
								    176,
								    183,
								    270,
								    11,
								    262,
								    42,
								    61,
								    265,
								]

								self.assertListEqual(_lowercase      ,       self.big_tokenizer.encode(_lowercase))



					@require_torch
					@slow
					def     __snake_case						(	self     :						List[str])    ->							Tuple:
								import torch

								from transformers import ReformerConfig, ReformerModel

								# Build sequence
								A_									= list(self.big_tokenizer.get_vocab().keys())[:10]
								A_									= ' '.join(_lowercase)
								A_									= self.big_tokenizer.encode_plus(_lowercase      ,       return_tensors='pt')
								A_									= self.big_tokenizer.batch_encode_plus([sequence, sequence]      ,       return_tensors='pt')

								A_									= ReformerConfig()
								# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
								A_									= encoded_sequence['input_ids'].shape
								A_									= ReformerModel(_lowercase)

								# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
								assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

								with torch.no_grad():
											model(**_lowercase)
											model(**_lowercase)



					@slow
					def     __snake_case						(	self     :						str)    ->							Union[str, Any]:
								# fmt: off
								A_									= {'input_ids': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								# This tokenizer does not know some characters like ")".
								# That is the reason why we use very simple texts here.
								# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
								A_									= [
								    'This is a very simple sentence.',
								    'The quick brown fox jumps over the lazy dog.',
								]

								self.tokenizer_integration_test_util(
								    expected_encoding=_lowercase      ,       model_name='google/reformer-crime-and-punishment'      ,       revision='0e6c3decb8211d49bf881013425dc8b0448b3f5a'      ,       padding=_lowercase      ,       sequences=_lowercase      ,       )




