




import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase   :			Union[str, Any]          = get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    ReformerTokenizer
							UpperCamelCase__   	=    ReformerTokenizerFast
							UpperCamelCase__   	=    True
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    True
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								super().setUp()

								a									=				ReformerTokenizer(__magic_name__						, keep_accents=__magic_name__							)
								tokenizer.save_pretrained(self.tmpdirname							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				"""<s>"""
								a									=				1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__							)						, __magic_name__							)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__							)						, __magic_name__							)
							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				list(self.get_tokenizer().get_vocab().keys()							)

								self.assertEqual(vocab_keys[0]						, """<unk>"""							)
								self.assertEqual(vocab_keys[1]						, """<s>"""							)
								self.assertEqual(vocab_keys[-1]						, """j"""							)
								self.assertEqual(len(__magic_name__							)						, 1000							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								self.assertEqual(self.get_tokenizer().vocab_size						, 1000							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								if not self.test_rust_tokenizer:
									return

								a									=				self.get_tokenizer()
								a									=				self.get_rust_tokenizer()

								a									=				"""I was born in 92000, and this is falsé."""

								a									=				tokenizer.tokenize(__magic_name__							)
								a									=				rust_tokenizer.tokenize(__magic_name__							)
								self.assertListEqual(__magic_name__						, __magic_name__							)

								a									=				tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
								a									=				rust_tokenizer.encode(__magic_name__						, add_special_tokens=__magic_name__							)
								self.assertListEqual(__magic_name__						, __magic_name__							)

								a									=				self.get_rust_tokenizer()
								a									=				tokenizer.encode(__magic_name__							)
								a									=				rust_tokenizer.encode(__magic_name__							)
								self.assertListEqual(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:int						, __magic_name__		:List[str]=15							):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
										a									=				self.rust_tokenizer_class.from_pretrained(__magic_name__						, **__magic_name__							)

										# Simple input
										a									=				"""This is a simple input"""
										a									=				["""This is a simple input 1""", """This is a simple input 2"""]
										a									=				("""This is a simple input""", """This is a pair""")
										a									=				[
										    ("""This is a simple input 1""", """This is a simple input 2"""),
										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
										]

										# Simple input tests
										self.assertRaises(__magic_name__						, tokenizer_r.encode						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""							)

										# Simple input
										self.assertRaises(__magic_name__						, tokenizer_r.encode_plus						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""							)

										# Simple input
										self.assertRaises(
										    __magic_name__						, tokenizer_r.batch_encode_plus						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""						, )

										# Pair input
										self.assertRaises(__magic_name__						, tokenizer_r.encode						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""							)

										# Pair input
										self.assertRaises(__magic_name__						, tokenizer_r.encode_plus						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""							)

										# Pair input
										self.assertRaises(
										    __magic_name__						, tokenizer_r.batch_encode_plus						, __magic_name__						, max_length=__magic_name__						, padding="""max_length"""						, )
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								pass
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a									=				ReformerTokenizer(__magic_name__						, keep_accents=__magic_name__							)

								a									=				tokenizer.tokenize("""This is a test"""							)
								self.assertListEqual(__magic_name__						, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(__magic_name__							)						, [285, 46, 10, 170, 382]						, )

								a									=				tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
								self.assertListEqual(
								    __magic_name__						, [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]						, )
								a									=				tokenizer.convert_tokens_to_ids(__magic_name__							)
								self.assertListEqual(
								    __magic_name__						, [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						, )

								a									=				tokenizer.convert_ids_to_tokens(__magic_name__							)
								self.assertListEqual(
								    __magic_name__						, [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]						, )
							@cached_property
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""							)
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								a									=				"""Hello World!"""
								a									=				[126, 32, 262, 152, 38, 72, 287]

								self.assertListEqual(__magic_name__						, self.big_tokenizer.encode(__magic_name__							)							)
							@slow
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				(
								    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
								    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
								)
								a									=				[
								    108,
								    265,
								    24,
								    111,
								    4,
								    258,
								    156,
								    35,
								    28,
								    275,
								    3,
								    259,
								    297,
								    260,
								    84,
								    4,
								    35,
								    110,
								    44,
								    8,
								    259,
								    91,
								    268,
								    21,
								    11,
								    209,
								    274,
								    109,
								    266,
								    277,
								    117,
								    86,
								    93,
								    315,
								    258,
								    278,
								    258,
								    277,
								    258,
								    0,
								    258,
								    288,
								    258,
								    319,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    287,
								    258,
								    315,
								    258,
								    289,
								    258,
								    278,
								    99,
								    269,
								    266,
								    262,
								    8,
								    259,
								    241,
								    4,
								    217,
								    230,
								    268,
								    266,
								    55,
								    168,
								    106,
								    75,
								    193,
								    266,
								    223,
								    27,
								    49,
								    26,
								    282,
								    25,
								    264,
								    299,
								    19,
								    26,
								    0,
								    258,
								    277,
								    117,
								    86,
								    93,
								    176,
								    183,
								    270,
								    11,
								    262,
								    42,
								    61,
								    265,
								]

								self.assertListEqual(__magic_name__						, self.big_tokenizer.encode(__magic_name__							)							)
							@require_torch
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								import torch

								from transformers import ReformerConfig, ReformerModel

								# Build sequence
								a									=				list(self.big_tokenizer.get_vocab().keys()							)[:10]
								a									=				""" """.join(__magic_name__							)
								a									=				self.big_tokenizer.encode_plus(__magic_name__						, return_tensors="""pt"""							)
								a									=				self.big_tokenizer.batch_encode_plus([sequence, sequence]						, return_tensors="""pt"""							)

								a									=				ReformerConfig()
								# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
								a									=				encoded_sequence["""input_ids"""].shape
								a									=				ReformerModel(__magic_name__							)

								# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
								assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

								with torch.no_grad():
									model(**__magic_name__							)
									model(**__magic_name__							)





							@slow
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				{"""input_ids""": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								# This tokenizer does not know some characters like ")".
								# That is the reason why we use very simple texts here.
								# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
								a									=				[
								    """This is a very simple sentence.""",
								    """The quick brown fox jumps over the lazy dog.""",
								]

								self.tokenizer_integration_test_util(
								    expected_encoding=__magic_name__						, model_name="""google/reformer-crime-and-punishment"""						, revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""						, padding=__magic_name__						, sequences=__magic_name__						, )







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase   :			Union[str, Any]          = logging.get_logger(__name__)

__UpperCamelCase   :			Optional[Any]          = "▁"

__UpperCamelCase   :			Any          = {"vocab_file": "sentencepiece.bpe.model"}

__UpperCamelCase   :			int          = {
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

__UpperCamelCase   :			Dict          = {
    "facebook/xglm-564M": 2_048,
}

class 						__lowerCAmelCase      (      __magic_name__					):
							UpperCamelCase__   	=    VOCAB_FILES_NAMES
							UpperCamelCase__   	=    PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__   	=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__   	=    ['''input_ids''', '''attention_mask''']
							def __init__(		self		:Optional[Any]						, __magic_name__		:List[str]						, __magic_name__		:Tuple="<s>"						, __magic_name__		:List[Any]="</s>"						, __magic_name__		:List[str]="</s>"						, __magic_name__		:Any="<s>"						, __magic_name__		:Tuple="<unk>"						, __magic_name__		:int="<pad>"						, __magic_name__		:Optional[Dict[str, Any]] = None						, **__magic_name__		:Optional[Any]						, ):







								'''simple docstring'''







								a									=				{} if sp_model_kwargs is None else sp_model_kwargs

								# Compatibility with the original tokenizer
								a									=				7
								a									=				[F'<madeupword{i}>' for i in range(self.num_madeup_words							)]

								a									=				kwargs.get("""additional_special_tokens"""						, []							)
								kwargs["additional_special_tokens"] += [
								    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    bos_token=__magic_name__						, eos_token=__magic_name__						, unk_token=__magic_name__						, sep_token=__magic_name__						, cls_token=__magic_name__						, pad_token=__magic_name__						, sp_model_kwargs=self.sp_model_kwargs						, **__magic_name__						, )

								a									=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(str(__magic_name__							)							)
								a									=				vocab_file

								# Original fairseq vocab and spm vocab must be "aligned":
								# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
								# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
								# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
								# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

								# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
								a									=				1

								# Mimic fairseq token-to-id alignment for the first 4 token
								a									=				{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

								a									=				len(self.sp_model							)
								a									=				{F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words							)}
								self.fairseq_tokens_to_ids.update(__magic_name__							)

								a									=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							def __getstate__(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.__dict__.copy()
								a									=				None
								a									=				self.sp_model.serialized_model_proto()
								return state
							def __setstate__(		self		:Any						, __magic_name__		:Any							):







								'''simple docstring'''







								a									=				d

								# for backward compatibility
								if not hasattr(self						, """sp_model_kwargs"""							):
									a									=				{}

								a									=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None							):







								'''simple docstring'''







								if token_ids_a is None:
									return [self.sep_token_id] + token_ids_a
								a									=				[self.sep_token_id]
								return sep + token_ids_a + sep + sep + token_ids_a
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None						, __magic_name__		:bool = False							):







								'''simple docstring'''







								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__magic_name__						, token_ids_a=__magic_name__						, already_has_special_tokens=__magic_name__							)

								if token_ids_a is None:
									return [1] + ([0] * len(__magic_name__							))
								return [1] + ([0] * len(__magic_name__							)) + [1, 1] + ([0] * len(__magic_name__							))
							def  lowerCamelCase__				(		self		:Optional[Any]						, __magic_name__		:List[int]						, __magic_name__		:Optional[List[int]] = None							):







								'''simple docstring'''







								a									=				[self.sep_token_id]

								if token_ids_a is None:
									return len(sep + token_ids_a							) * [0]
								return len(sep + token_ids_a + sep + sep + token_ids_a							) * [0]
							@property
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								return len(self.sp_model							) + self.fairseq_offset + self.num_madeup_words
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				{self.convert_ids_to_tokens(__magic_name__							): i for i in range(self.vocab_size							)}
								vocab.update(self.added_tokens_encoder							)
								return vocab
							def  lowerCamelCase__				(		self		:Any						, __magic_name__		:str							):







								'''simple docstring'''







								return self.sp_model.encode(__magic_name__						, out_type=__magic_name__							)
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:Any							):







								'''simple docstring'''







								if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
								a									=				self.sp_model.PieceToId(__magic_name__							)

								# Need to return unknown token if the SP model returned 0
								return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:Dict							):







								'''simple docstring'''







								if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
								return self.sp_model.IdToPiece(index - self.fairseq_offset							)
							def  lowerCamelCase__				(		self		:List[str]						, __magic_name__		:List[str]							):







								'''simple docstring'''







								a									=				"""""".join(__magic_name__							).replace(__magic_name__						, """ """							).strip()
								return out_string





							def  lowerCamelCase__				(		self		:List[Any]						, __magic_name__		:str						, __magic_name__		:Optional[str] = None							):







								'''simple docstring'''







								if not os.path.isdir(__magic_name__							):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
									return
								a									=				os.path.join(
								    __magic_name__						, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(__magic_name__							) and os.path.isfile(self.vocab_file							):
									copyfile(self.vocab_file						, __magic_name__							)
								elif not os.path.isfile(self.vocab_file							):
									with open(__magic_name__						, """wb"""							) as fi:
										a									=				self.sp_model.serialized_model_proto()
										fi.write(__magic_name__							)

								return (out_vocab_file,)

