



'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_	 =						get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
@require_tokenizers
class __A    (						A     ,     unittest.TestCase		):



 '''simple docstring'''







 __lowerCamelCase					: Optional[Any]    = XLNetTokenizer
 __lowerCamelCase					: Tuple    = XLNetTokenizerFast
 __lowerCamelCase					: int    = True
 __lowerCamelCase					: int    = True



 def       a__						(self )					->  Optional[int]:


      """simple docstring"""



      super().setUp()

      # We have a SentencePiece fixture for testing
      _a											=						XLNetTokenizer(A				,     keep_accents=A )
      tokenizer.sanitize_special_tokens()
      tokenizer.save_pretrained(self.tmpdirname )



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						'''<s>'''
      _a											=						1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(A )				,     A )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(A )				,     A )



 def       a__						(self )					->  List[str]:


      """simple docstring"""



      _a											=						list(self.get_tokenizer().get_vocab().keys() )

      self.assertEqual(vocab_keys[0]				,     '''<unk>''' )
      self.assertEqual(vocab_keys[1]				,     '''<s>''' )
      self.assertEqual(vocab_keys[-1]				,     '''<eod>''' )
      self.assertEqual(len(A )				,     1_006 )



 def       a__						(self )					->  Tuple:


      """simple docstring"""



      self.assertEqual(self.get_tokenizer().vocab_size				,     1_000 )



 def       a__						(self )					->  Dict:


      """simple docstring"""



      _a											=						XLNetTokenizer(A				,     keep_accents=A )

      _a											=						tokenizer.tokenize('''This is a test''' )
      self.assertListEqual(A				,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(A )				,     [285, 46, 10, 170, 382] )

      _a											=						tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
      self.assertListEqual(
          A				,     [
              SPIECE_UNDERLINE + '''I''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''9''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''s''',
              '''é''',
              '''.''',
          ]				,     )
      _a											=						tokenizer.convert_tokens_to_ids(A )
      self.assertListEqual(A				,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

      _a											=						tokenizer.convert_ids_to_tokens(A )
      self.assertListEqual(
          A				,     [
              SPIECE_UNDERLINE + '''I''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''<unk>''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''s''',
              '''<unk>''',
              '''.''',
          ]				,     )



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						XLNetTokenizer(A				,     do_lower_case=A )
      _a											=						tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
      self.assertListEqual(
          A				,     [
              SPIECE_UNDERLINE + '''''',
              '''i''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''9''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''se''',
              '''.''',
          ]				,     )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )				,     ['''▁he''', '''ll''', '''o'''] )



 def       a__						(self )					->  List[str]:


      """simple docstring"""



      _a											=						XLNetTokenizer(A				,     do_lower_case=A )
      _a											=						tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
      self.assertListEqual(
          A				,     [
              SPIECE_UNDERLINE + '''I''',
              SPIECE_UNDERLINE + '''was''',
              SPIECE_UNDERLINE + '''b''',
              '''or''',
              '''n''',
              SPIECE_UNDERLINE + '''in''',
              SPIECE_UNDERLINE + '''''',
              '''9''',
              '''2''',
              '''0''',
              '''0''',
              '''0''',
              ''',''',
              SPIECE_UNDERLINE + '''and''',
              SPIECE_UNDERLINE + '''this''',
              SPIECE_UNDERLINE + '''is''',
              SPIECE_UNDERLINE + '''f''',
              '''al''',
              '''se''',
              '''.''',
          ]				,     )



 @slow
 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						XLNetTokenizer.from_pretrained('''xlnet-base-cased''' )

      _a											=						tokenizer.encode('''sequence builders'''				,     add_special_tokens=A )
      _a											=						tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=A )

      _a											=						tokenizer.build_inputs_with_special_tokens(A )
      _a											=						tokenizer.build_inputs_with_special_tokens(A				,     A )

      assert encoded_sentence == text + [4, 3]
      assert encoded_pair == text + [4] + text_a + [4, 3]



 @slow
 def       a__						(self )					->  Dict:


      """simple docstring"""



      _a											=						{'''input_ids''': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=A				,     model_name='''xlnet-base-cased'''				,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				,     )








'''simple docstring'''





import PIL.Image
import PIL.ImageOps
from packaging import version
from PIL import Image


if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):
  lowercase_	 =						{
      "linear": PIL.Image.Resampling.BILINEAR,
      "bilinear": PIL.Image.Resampling.BILINEAR,
      "bicubic": PIL.Image.Resampling.BICUBIC,
      "lanczos": PIL.Image.Resampling.LANCZOS,
      "nearest": PIL.Image.Resampling.NEAREST,
  }
else:
  lowercase_	 =						{
      "linear": PIL.Image.LINEAR,
      "bilinear": PIL.Image.BILINEAR,
      "bicubic": PIL.Image.BICUBIC,
      "lanczos": PIL.Image.LANCZOS,
      "nearest": PIL.Image.NEAREST,
  }




def lowerCAmelCase						(__A):



     """simple docstring"""



     _a											=						(images / 2 + 0.5).clamp(0						,			1)
     _a											=						images.cpu().permute(0						,			2						,			3						,			1).float().numpy()
     _a											=						numpy_to_pil(__A)
     return images




def lowerCAmelCase						(__A):



     """simple docstring"""



     if images.ndim == 3:
          _a											=						images[None, ...]
     _a											=						(images * 255).round().astype('''uint8''')
     if images.shape[-1] == 1:
          # special case for grayscale (single channel) images
          _a											=						[Image.fromarray(image.squeeze()						,			mode='''L''') for image in images]
     else:
          _a											=						[Image.fromarray(__A) for image in images]

     return pil_images



