
'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE     =       get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
@require_tokenizers
class 							UpperCAmelCase_							(    A					,				unittest.TestCase				):

 '''simple docstring'''
 lowercase_					:    str				=							XLNetTokenizer
 lowercase_					:    List[str]				=							XLNetTokenizerFast
 lowercase_					:    Optional[Any]				=							True
 lowercase_					:    Optional[int]				=							True







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   super().setUp()

   # We have a SentencePiece fixture for testing
   UpperCAmelCase__      :					Optional[Any]				       =							XLNetTokenizer(snake_case__			,				keep_accents=snake_case__ )
   tokenizer.sanitize_special_tokens()
   tokenizer.save_pretrained(self.tmpdirname )







 def 							UpperCamelCase  (					self		:  Optional[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							"<s>"
   UpperCAmelCase__      :					Optional[Any]				       =							1

   self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__ )			,				snake_case__ )
   self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__ )			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[str]				       =							list(self.get_tokenizer().get_vocab().keys() )

   self.assertEqual(vocab_keys[0]			,				"<unk>" )
   self.assertEqual(vocab_keys[1]			,				"<s>" )
   self.assertEqual(vocab_keys[-1]			,				"<eod>" )
   self.assertEqual(len(snake_case__ )			,				10_06 )







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   self.assertEqual(self.get_tokenizer().vocab_size			,				10_00 )







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							XLNetTokenizer(snake_case__			,				keep_accents=snake_case__ )

   UpperCAmelCase__      :					Optional[int]				       =							tokenizer.tokenize("This is a test" )
   self.assertListEqual(snake_case__			,				["▁This", "▁is", "▁a", "▁t", "est"] )

   self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )			,				[2_85, 46, 10, 1_70, 3_82] )

   UpperCAmelCase__      :					Union[str, Any]				       =							tokenizer.tokenize("I was born in 92000, and this is falsé." )
   self.assertListEqual(
       snake_case__			,				[
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "s",
           "é",
           ".",
       ]			,				)
   UpperCAmelCase__      :					int				       =							tokenizer.convert_tokens_to_ids(snake_case__ )
   self.assertListEqual(snake_case__			,				[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

   UpperCAmelCase__      :					int				       =							tokenizer.convert_ids_to_tokens(snake_case__ )
   self.assertListEqual(
       snake_case__			,				[
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "<unk>",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "s",
           "<unk>",
           ".",
       ]			,				)







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							XLNetTokenizer(snake_case__			,				do_lower_case=snake_case__ )
   UpperCAmelCase__      :					Optional[Any]				       =							tokenizer.tokenize("I was born in 92000, and this is falsé." )
   self.assertListEqual(
       snake_case__			,				[
           SPIECE_UNDERLINE + "",
           "i",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "se",
           ".",
       ]			,				)
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )			,				["▁he", "ll", "o"] )







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[Any]				       =							XLNetTokenizer(snake_case__			,				do_lower_case=snake_case__ )
   UpperCAmelCase__      :					Dict				       =							tokenizer.tokenize("I was born in 92000, and this is falsé." )
   self.assertListEqual(
       snake_case__			,				[
           SPIECE_UNDERLINE + "I",
           SPIECE_UNDERLINE + "was",
           SPIECE_UNDERLINE + "b",
           "or",
           "n",
           SPIECE_UNDERLINE + "in",
           SPIECE_UNDERLINE + "",
           "9",
           "2",
           "0",
           "0",
           "0",
           ",",
           SPIECE_UNDERLINE + "and",
           SPIECE_UNDERLINE + "this",
           SPIECE_UNDERLINE + "is",
           SPIECE_UNDERLINE + "f",
           "al",
           "se",
           ".",
       ]			,				)







 @slow
 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							XLNetTokenizer.from_pretrained("xlnet-base-cased" )

   UpperCAmelCase__      :					List[Any]				       =							tokenizer.encode("sequence builders"			,				add_special_tokens=snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							tokenizer.encode("multi-sequence build"			,				add_special_tokens=snake_case__ )

   UpperCAmelCase__      :					Any				       =							tokenizer.build_inputs_with_special_tokens(snake_case__ )
   UpperCAmelCase__      :					List[Any]				       =							tokenizer.build_inputs_with_special_tokens(snake_case__			,				snake_case__ )

   assert encoded_sentence == text + [4, 3]
   assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							{"input_ids": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
   # fmt: on

   self.tokenizer_integration_test_util(
       expected_encoding=snake_case__			,				model_name="xlnet-base-cased"			,				revision="c841166438c31ec7ca9a106dee7bb312b73ae511"			,				)





'''simple docstring'''





from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, apply_forward_hook
from .modeling_utils import ModelMixin
from .vae import Decoder, DecoderOutput, Encoder, VectorQuantizer






@dataclass
class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    torch.FloatTensor






class 							UpperCAmelCase_							(    A					,				A				):

 '''simple docstring'''







 @register_to_config
 def __init__(					self		:  List[str]			,				snake_case__		:  int = 3			,				snake_case__		:  int = 3			,				snake_case__		:  Tuple[str] = ("DownEncoderBlock2D",)			,				snake_case__		:  Tuple[str] = ("UpDecoderBlock2D",)			,				snake_case__		:  Tuple[int] = (64,)			,				snake_case__		:  int = 1			,				snake_case__		:  str = "silu"			,				snake_case__		:  int = 3			,				snake_case__		:  int = 32			,				snake_case__		:  int = 2_56			,				snake_case__		:  int = 32			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  float = 0.18215			,				snake_case__		:  str = "group"			,				):



   '''simple docstring'''




   super().__init__()

   # pass init params to Encoder
   UpperCAmelCase__      :					Tuple				       =							Encoder(
       in_channels=snake_case__			,				out_channels=snake_case__			,				down_block_types=snake_case__			,				block_out_channels=snake_case__			,				layers_per_block=snake_case__			,				act_fn=snake_case__			,				norm_num_groups=snake_case__			,				double_z=snake_case__			,				)

   UpperCAmelCase__      :					str				       =							vq_embed_dim if vq_embed_dim is not None else latent_channels

   UpperCAmelCase__      :					Tuple				       =							nn.Convad(snake_case__			,				snake_case__			,				1 )
   UpperCAmelCase__      :					int				       =							VectorQuantizer(snake_case__			,				snake_case__			,				beta=0.25			,				remap=snake_case__			,				sane_index_shape=snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							nn.Convad(snake_case__			,				snake_case__			,				1 )

   # pass init params to Decoder
   UpperCAmelCase__      :					Union[str, Any]				       =							Decoder(
       in_channels=snake_case__			,				out_channels=snake_case__			,				up_block_types=snake_case__			,				block_out_channels=snake_case__			,				layers_per_block=snake_case__			,				act_fn=snake_case__			,				norm_num_groups=snake_case__			,				norm_type=snake_case__			,				)







 @apply_forward_hook
 def 							UpperCamelCase  (					self		:  Union[str, Any]			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[Any]				       =							self.encoder(snake_case__ )
   UpperCAmelCase__      :					List[str]				       =							self.quant_conv(snake_case__ )

   if not return_dict:
     return (h,)

   return VQEncoderOutput(latents=snake_case__ )







 @apply_forward_hook
 def 							UpperCamelCase  (					self		:  List[str]			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = False			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   if not force_not_quantize:
     UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					Any				       =							self.quantize(snake_case__ )
   else:
     UpperCAmelCase__      :					Tuple				       =							h
   UpperCAmelCase__      :					int				       =							self.post_quant_conv(snake_case__ )
   UpperCAmelCase__      :					Optional[int]				       =							self.decoder(snake_case__			,				quant if self.config.norm_type == "spatial" else None )

   if not return_dict:
     return (dec,)

   return DecoderOutput(sample=snake_case__ )







 def 							UpperCamelCase  (					self		:  Any			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							sample
   UpperCAmelCase__      :					int				       =							self.encode(snake_case__ ).latents
   UpperCAmelCase__      :					Any				       =							self.decode(snake_case__ ).sample

   if not return_dict:
     return (dec,)

   return DecoderOutput(sample=snake_case__ )



