






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_	:    Dict            =      get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
class   UpperCamelCase_ (       a_   ,					unittest.TestCase  ):
  _A       :					str				=	XLMProphetNetTokenizer
  _A       :					List[Any]				=	False
  _A       :					Tuple				=	True



  def 		UpperCamelCase_				(      self				)			->			Any:



         """simple docstring"""





         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase	    =			XLMProphetNetTokenizer(snake_case__ ,		keep_accents=snake_case__				)
         tokenizer.save_pretrained(self.tmpdirname				)



  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			"""[PAD]"""
         UpperCAmelCase	    =			0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__				) ,		snake_case__				)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__				) ,		snake_case__				)



  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			list(self.get_tokenizer().get_vocab().keys()				)

         self.assertEqual(vocab_keys[0] ,		"""[PAD]"""				)
         self.assertEqual(vocab_keys[1] ,		"""[CLS]"""				)
         self.assertEqual(vocab_keys[-1] ,		"""j"""				)
         self.assertEqual(len(snake_case__				) ,		10_12				)



  def 		UpperCamelCase_				(      self				)			->			Any:



         """simple docstring"""





         self.assertEqual(self.get_tokenizer().vocab_size ,		10_12				)



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			XLMProphetNetTokenizer(snake_case__ ,		keep_accents=snake_case__				)

         UpperCAmelCase	    =			tokenizer.tokenize("""This is a test"""				)
         self.assertListEqual(snake_case__ ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(snake_case__				) ,		[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]] ,		)

         UpperCAmelCase	    =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
         self.assertListEqual(
             snake_case__ ,		[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ] ,		)
         UpperCAmelCase	    =			tokenizer.convert_tokens_to_ids(snake_case__				)
         self.assertListEqual(
             snake_case__ ,		[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
             ] ,		)

         UpperCAmelCase	    =			tokenizer.convert_ids_to_tokens(snake_case__				)
         self.assertListEqual(
             snake_case__ ,		[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """[UNK]""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """[UNK]""",
                 """.""",
             ] ,		)



  @cached_property
  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""				)



  @slow
  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			"""Hello World!"""
         UpperCAmelCase	    =			[3_53_89, 66_72, 49, 2]
         self.assertListEqual(snake_case__ ,		self.big_tokenizer.encode(snake_case__				)				)



  @slow
  def 		UpperCamelCase_				(      self				)			->			Dict:



         """simple docstring"""





         UpperCAmelCase	    =			{"""input_ids""": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=snake_case__ ,		model_name="""microsoft/xprophetnet-large-wiki100-cased""" ,		revision="""1acad1643ddd54a44df6a1b797ada8373685d90e""" ,		)








"""simple docstring"""







from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_	:    str            =      logging.get_logger(__name__)

lowerCAmelCase_	:    Optional[int]            =      {
    '''huggingface/time-series-transformer-tourism-monthly''': (
        '''https://huggingface.co/huggingface/time-series-transformer-tourism-monthly/resolve/main/config.json'''
    ),
    # See all TimeSeriesTransformer models at https://huggingface.co/models?filter=time_series_transformer
}





class   UpperCamelCase_ (       a_  ):
  _A       :					int				=	'time_series_transformer'
  _A       :					List[Any]				=	{
      'hidden_size': 'd_model',
      'num_attention_heads': 'encoder_attention_heads',
      'num_hidden_layers': 'encoder_layers',
  }



  def __init__(      self ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = "student_t" ,		snake_case__ = "nll" ,		snake_case__ = 1 ,		snake_case__ = [1, 2, 3, 4, 5, 6, 7] ,		snake_case__ = "mean" ,		snake_case__ = 0 ,		snake_case__ = 0 ,		snake_case__ = 0 ,		snake_case__ = 0 ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = 32 ,		snake_case__ = 32 ,		snake_case__ = 2 ,		snake_case__ = 2 ,		snake_case__ = 2 ,		snake_case__ = 2 ,		snake_case__ = True ,		snake_case__ = "gelu" ,		snake_case__ = 64 ,		snake_case__ = 0.1 ,		snake_case__ = 0.1 ,		snake_case__ = 0.1 ,		snake_case__ = 0.1 ,		snake_case__ = 0.1 ,		snake_case__ = 1_00 ,		snake_case__ = 0.02 ,		snake_case__=True ,		**snake_case__ ,		)			->			Union[str, Any]:



         """simple docstring"""





         UpperCAmelCase	    =			prediction_length
         UpperCAmelCase	    =			context_length or prediction_length
         UpperCAmelCase	    =			distribution_output
         UpperCAmelCase	    =			loss
         UpperCAmelCase	    =			input_size
         UpperCAmelCase	    =			num_time_features
         UpperCAmelCase	    =			lags_sequence
         UpperCAmelCase	    =			scaling
         UpperCAmelCase	    =			num_dynamic_real_features
         UpperCAmelCase	    =			num_static_real_features
         UpperCAmelCase	    =			num_static_categorical_features
         if cardinality and num_static_categorical_features > 0:
                if len(snake_case__				) != num_static_categorical_features:
                       raise ValueError(
                           """The cardinality should be a list of the same length as `num_static_categorical_features`"""				)
                UpperCAmelCase	    =			cardinality
         else:
                UpperCAmelCase	    =			[0]
         if embedding_dimension and num_static_categorical_features > 0:
                if len(snake_case__				) != num_static_categorical_features:
                       raise ValueError(
                           """The embedding dimension should be a list of the same length as `num_static_categorical_features`"""				)
                UpperCAmelCase	    =			embedding_dimension
         else:
                UpperCAmelCase	    =			[min(50 ,		(cat + 1) // 2				) for cat in self.cardinality]
         UpperCAmelCase	    =			num_parallel_samples

         # Transformer architecture configuration
         UpperCAmelCase	    =			input_size * len(snake_case__				) + self._number_of_features
         UpperCAmelCase	    =			d_model
         UpperCAmelCase	    =			encoder_attention_heads
         UpperCAmelCase	    =			decoder_attention_heads
         UpperCAmelCase	    =			encoder_ffn_dim
         UpperCAmelCase	    =			decoder_ffn_dim
         UpperCAmelCase	    =			encoder_layers
         UpperCAmelCase	    =			decoder_layers

         UpperCAmelCase	    =			dropout
         UpperCAmelCase	    =			attention_dropout
         UpperCAmelCase	    =			activation_dropout
         UpperCAmelCase	    =			encoder_layerdrop
         UpperCAmelCase	    =			decoder_layerdrop

         UpperCAmelCase	    =			activation_function
         UpperCAmelCase	    =			init_std

         UpperCAmelCase	    =			use_cache

         super().__init__(is_encoder_decoder=snake_case__ ,		**snake_case__				)



  @property
  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return (
             sum(self.embedding_dimension				)
             + self.num_dynamic_real_features
             + self.num_time_features
             + self.num_static_real_features
             + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
         )
