




"""simple docstring"""







from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_	     =       logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_	     =       {
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
SCREAMING_SNAKE_CASE_	     =       1114112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
SCREAMING_SNAKE_CASE_	     =       0

SCREAMING_SNAKE_CASE_	     =       0Xe_000
SCREAMING_SNAKE_CASE_	     =       0Xe_001
SCREAMING_SNAKE_CASE_	     =       0Xe_002
SCREAMING_SNAKE_CASE_	     =       0Xe_003
SCREAMING_SNAKE_CASE_	     =       0Xe_004

# Maps special codepoints to human-readable names.
SCREAMING_SNAKE_CASE_	     =       {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
SCREAMING_SNAKE_CASE_	     =       {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   A_				      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







   def __init__(		self       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=chr(lowerCamelCase_)       ,					lowerCamelCase_=False       ,					lowerCamelCase_=2_0_4_8       ,					**lowerCamelCase_       ,					)				->   List[Any]:
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else bos_token
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else eos_token
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else sep_token
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else cls_token
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       UpperCamelCase              =	AddedToken(lowerCamelCase_       ,					lstrip=lowerCamelCase_       ,					rstrip=lowerCamelCase_) if isinstance(lowerCamelCase_       ,					lowerCamelCase_) else mask_token

       super().__init__(
           bos_token=lowerCamelCase_       ,					eos_token=lowerCamelCase_       ,					sep_token=lowerCamelCase_       ,					cls_token=lowerCamelCase_       ,					pad_token=lowerCamelCase_       ,					mask_token=lowerCamelCase_       ,					add_prefix_space=lowerCamelCase_       ,					model_max_length=lowerCamelCase_       ,					**lowerCamelCase_       ,					)

       # Creates a mapping for looking up the IDs of special symbols.
       UpperCamelCase              =	{}
       for codepoint, name in SPECIAL_CODEPOINTS.items():
           UpperCamelCase              =	codepoint

       # Creates a mapping for looking up the string forms of special symbol IDs.
       UpperCamelCase              =	{
           codepoint: name for name, codepoint in self._special_codepoints.items()
       }

       UpperCamelCase              =	UNICODE_VOCAB_SIZE
       UpperCamelCase              =	len(self._special_codepoints)







   @property
   def   UpperCAmelCase__			(		self)				->   int:
       return self._unicode_vocab_size







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   List[str]:
       return list(lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   int:
       try:
           return ord(lowerCamelCase_)
       except TypeError:
           raise ValueError(F'invalid token: \'{token}\'')







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   str:
       try:
           if index in SPECIAL_CODEPOINTS:
               return SPECIAL_CODEPOINTS[index]
           return chr(lowerCamelCase_)
       except TypeError:
           raise ValueError(F'invalid id: {index}')







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Optional[int]:
       return "".join(lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	[self.sep_token_id]
       UpperCamelCase              =	[self.cls_token_id]

       UpperCamelCase              =	cls + token_ids_a + sep
       if token_ids_a is not None:
           result += token_ids_a + sep
       return result







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None       ,					lowerCamelCase_ = False)				->   List[int]:
       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=lowerCamelCase_       ,					token_ids_a=lowerCamelCase_       ,					already_has_special_tokens=lowerCamelCase_)

       UpperCamelCase              =	[1] + ([0] * len(lowerCamelCase_)) + [1]
       if token_ids_a is not None:
           result += ([0] * len(lowerCamelCase_)) + [1]
       return result







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	[self.sep_token_id]
       UpperCamelCase              =	[self.cls_token_id]

       UpperCamelCase              =	len(cls + token_ids_a + sep) * [0]
       if token_ids_a is not None:
           result += len(token_ids_a + sep) * [1]
       return result







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[str]:
       return ()






"""simple docstring"""



import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_					=     get_tests_dir("""fixtures/spiece.model""")






@require_sentencepiece
@require_tokenizers
class 						__snake_case   (						SCREAMING_SNAKE_CASE__				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	AlbertTokenizer
 _lowerCamelCase								=	AlbertTokenizerFast
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     super().setUp()

     # We have a SentencePiece fixture for testing
     __A       :		str							=	AlbertTokenizer(__lowerCamelCase    )
     tokenizer.save_pretrained(self.tmpdirname    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     __A       :		Tuple							=	'''this is a test'''
     __A       :		Union[str, Any]							=	'''this is a test'''
     return input_text, output_text

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	'''<pad>'''
     __A       :		Optional[int]							=	0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase    )   ,		__lowerCamelCase    )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase    )   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	list(self.get_tokenizer().get_vocab().keys()    )

     self.assertEqual(vocab_keys[0]   ,		'''<pad>'''    )
     self.assertEqual(vocab_keys[1]   ,		'''<unk>'''    )
     self.assertEqual(vocab_keys[-1]   ,		'''▁eloquent'''    )
     self.assertEqual(len(__lowerCamelCase    )   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     self.assertEqual(self.get_tokenizer().vocab_size   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     if not self.test_rust_tokenizer:
         return

     __A       :		Dict							=	self.get_tokenizer()
     __A       :		List[Any]							=	self.get_rust_tokenizer()

     __A       :		int							=	'''I was born in 92000, and this is falsé.'''

     __A       :		Any							=	tokenizer.tokenize(__lowerCamelCase    )
     __A       :		Dict							=	rust_tokenizer.tokenize(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     __A       :		List[str]							=	rust_tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	self.get_rust_tokenizer()
     __A       :		Union[str, Any]							=	tokenizer.encode(__lowerCamelCase    )
     __A       :		List[Any]							=	rust_tokenizer.encode(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	AlbertTokenizer(__lowerCamelCase   ,		keep_accents=__lowerCamelCase    )

     __A       :		Optional[int]							=	tokenizer.tokenize('''This is a test'''    )
     self.assertListEqual(__lowerCamelCase   ,		['''▁this''', '''▁is''', '''▁a''', '''▁test''']    )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase    )   ,		[48, 25, 21, 1289]    )

     __A       :		Tuple							=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']    )
     __A       :		Optional[Any]							=	tokenizer.convert_tokens_to_ids(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]    )

     __A       :		Optional[Any]							=	tokenizer.convert_ids_to_tokens(__lowerCamelCase    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']   ,		)

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	AlbertTokenizer(__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode('''sequence builders'''    )
     __A       :		str							=	tokenizer.encode('''multi-sequence build'''    )

     __A       :		Union[str, Any]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    )
     __A       :		Optional[int]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   ,		__lowerCamelCase    )

     assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
     assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
         tokenizer.sep_token_id
     ]







 @slow
 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Dict							=	{'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__lowerCamelCase   ,		model_name='''albert-base-v2'''   ,		revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''   ,		)




