

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


A__						=							logging.get_logger(__name__)


A__						=							{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all MVP models at https://huggingface.co/models?filter=mvp
A__						=							{
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json",
    },
}

A__						=							{
    "RUCAIBox/mvp": 1024,
}

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   List[str]       				=  VOCAB_FILES_NAMES
	_lowercase  :   str       				=  PRETRAINED_VOCAB_FILES_MAP
	_lowercase  :   Tuple       				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowercase  :   Any       				=  ["input_ids", "attention_mask"]
	_lowercase  :   Optional[int]       				=  MvpTokenizer







	def __init__(   self: int  , __UpperCamelCase: Union[str, Any]=None  , __UpperCamelCase: Optional[Any]=None  , __UpperCamelCase: Any=None  , __UpperCamelCase: Dict="replace"  , __UpperCamelCase: Dict="<s>"  , __UpperCamelCase: Union[str, Any]="</s>"  , __UpperCamelCase: List[Any]="</s>"  , __UpperCamelCase: List[Any]="<s>"  , __UpperCamelCase: int="<unk>"  , __UpperCamelCase: Any="<pad>"  , __UpperCamelCase: List[str]="<mask>"  , __UpperCamelCase: Optional[Any]=False  , __UpperCamelCase: List[Any]=True  , **__UpperCamelCase: List[str]  , ):

						'''simple docstring'''





						super().__init__(
						    __UpperCamelCase  , __UpperCamelCase  , tokenizer_file=__UpperCamelCase  , errors=__UpperCamelCase  , bos_token=__UpperCamelCase  , eos_token=__UpperCamelCase  , sep_token=__UpperCamelCase  , cls_token=__UpperCamelCase  , unk_token=__UpperCamelCase  , pad_token=__UpperCamelCase  , mask_token=__UpperCamelCase  , add_prefix_space=__UpperCamelCase  , trim_offsets=__UpperCamelCase  , **__UpperCamelCase  , )

						__magic_name__							=    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
						if pre_tok_state.get('add_prefix_space'  , __UpperCamelCase      ) != add_prefix_space:
											__magic_name__							=    getattr(__UpperCamelCase  , pre_tok_state.pop('type'      )      )
											__magic_name__							=    add_prefix_space
											__magic_name__							=    pre_tok_class(**__UpperCamelCase      )

						__magic_name__							=    add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						__magic_name__							=    'post_processor'
						__magic_name__							=    getattr(self.backend_tokenizer  , __UpperCamelCase  , __UpperCamelCase      )
						if tokenizer_component_instance:
											__magic_name__							=    json.loads(tokenizer_component_instance.__getstate__()      )

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
																__magic_name__							=    tuple(state['sep']      )
											if "cls" in state:
																__magic_name__							=    tuple(state['cls']      )

											__magic_name__							=    False

											if state.get('add_prefix_space'  , __UpperCamelCase      ) != add_prefix_space:
																__magic_name__							=    add_prefix_space
																__magic_name__							=    True

											if state.get('trim_offsets'  , __UpperCamelCase      ) != trim_offsets:
																__magic_name__							=    trim_offsets
																__magic_name__							=    True

											if changes_to_apply:
																__magic_name__							=    getattr(__UpperCamelCase  , state.pop('type'      )      )
																__magic_name__							=    component_class(**__UpperCamelCase      )
																setattr(self.backend_tokenizer  , __UpperCamelCase  , __UpperCamelCase      )







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						if self._mask_token is None:
											if self.verbose:
																logger.error('Using mask_token, but it is not set yet.'      )
											return None
						return str(self._mask_token      )







	@mask_token.setter
	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: List[str]      ):

						'''simple docstring'''





						__magic_name__							=    AddedToken(__UpperCamelCase  , lstrip=__UpperCamelCase  , rstrip=__UpperCamelCase      ) if isinstance(__UpperCamelCase  , __UpperCamelCase      ) else value
						__magic_name__							=    value







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , *__UpperCamelCase: Any  , **__UpperCamelCase: Dict      ):

						'''simple docstring'''





						__magic_name__							=    kwargs.get('is_split_into_words'  , __UpperCamelCase      )

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'      )

						return super()._batch_encode_plus(*__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , *__UpperCamelCase: Any  , **__UpperCamelCase: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    kwargs.get('is_split_into_words'  , __UpperCamelCase      )

						if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    'to use it with pretokenized inputs.'      )

						return super()._encode_plus(*__UpperCamelCase  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]  , __UpperCamelCase: str  , __UpperCamelCase: Optional[str] = None      ):

						'''simple docstring'''





						__magic_name__							=    self._tokenizer.model.save(__UpperCamelCase  , name=__UpperCamelCase      )
						return tuple(__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: Optional[int]  , __UpperCamelCase: Optional[Any]=None      ):

						'''simple docstring'''





						__magic_name__							=    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
											return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: List[int]  , __UpperCamelCase: Optional[List[int]] = None      ):

						'''simple docstring'''





						__magic_name__							=    [self.sep_token_id]
						__magic_name__							=    [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep      ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]







from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__						=							logging.get_logger(__name__)

A__						=							{
    "huggingface/time-series-transformer-tourism-monthly": (
        "https://huggingface.co/huggingface/time-series-transformer-tourism-monthly/resolve/main/config.json"
    ),
    # See all TimeSeriesTransformer models at https://huggingface.co/models?filter=time_series_transformer
}

class 				__UpperCamelCase				(					SCREAMING_SNAKE_CASE			):
	_lowercase  :   Optional[int]       				=  "time_series_transformer"
	_lowercase  :   Dict       				=  {
	    "hidden_size": "d_model",
	    "num_attention_heads": "encoder_attention_heads",
	    "num_hidden_layers": "encoder_layers",
	}







	def __init__(   self: str  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: str = "student_t"  , __UpperCamelCase: str = "nll"  , __UpperCamelCase: int = 1  , __UpperCamelCase: List[int] = [1, 2, 3, 4, 5, 6, 7]  , __UpperCamelCase: Optional[Union[str, bool]] = "mean"  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: int = 0  , __UpperCamelCase: Optional[List[int]] = None  , __UpperCamelCase: Optional[List[int]] = None  , __UpperCamelCase: int = 32  , __UpperCamelCase: int = 32  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: int = 2  , __UpperCamelCase: bool = True  , __UpperCamelCase: str = "gelu"  , __UpperCamelCase: int = 64  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: float = 0.1  , __UpperCamelCase: int = 1_00  , __UpperCamelCase: float = 0.02  , __UpperCamelCase: Any=True  , **__UpperCamelCase: Optional[Any]  , ):

						'''simple docstring'''





						__magic_name__							=    prediction_length
						__magic_name__							=    context_length or prediction_length
						__magic_name__							=    distribution_output
						__magic_name__							=    loss
						__magic_name__							=    input_size
						__magic_name__							=    num_time_features
						__magic_name__							=    lags_sequence
						__magic_name__							=    scaling
						__magic_name__							=    num_dynamic_real_features
						__magic_name__							=    num_static_real_features
						__magic_name__							=    num_static_categorical_features
						if cardinality and num_static_categorical_features > 0:
											if len(__UpperCamelCase      ) != num_static_categorical_features:
																raise ValueError(
																    'The cardinality should be a list of the same length as `num_static_categorical_features`'      )
											__magic_name__							=    cardinality
						else:
											__magic_name__							=    [0]
						if embedding_dimension and num_static_categorical_features > 0:
											if len(__UpperCamelCase      ) != num_static_categorical_features:
																raise ValueError(
																    'The embedding dimension should be a list of the same length as `num_static_categorical_features`'      )
											__magic_name__							=    embedding_dimension
						else:
											__magic_name__							=    [min(50  , (cat + 1) // 2      ) for cat in self.cardinality]
						__magic_name__							=    num_parallel_samples

						# Transformer architecture configuration
						__magic_name__							=    input_size * len(__UpperCamelCase      ) + self._number_of_features
						__magic_name__							=    d_model
						__magic_name__							=    encoder_attention_heads
						__magic_name__							=    decoder_attention_heads
						__magic_name__							=    encoder_ffn_dim
						__magic_name__							=    decoder_ffn_dim
						__magic_name__							=    encoder_layers
						__magic_name__							=    decoder_layers

						__magic_name__							=    dropout
						__magic_name__							=    attention_dropout
						__magic_name__							=    activation_dropout
						__magic_name__							=    encoder_layerdrop
						__magic_name__							=    decoder_layerdrop

						__magic_name__							=    activation_function
						__magic_name__							=    init_std

						__magic_name__							=    use_cache

						super().__init__(is_encoder_decoder=__UpperCamelCase  , **__UpperCamelCase      )







	@property
	def 	_SCREAMING_SNAKE_CASE							(   self: str      ):

						'''simple docstring'''





						return (
						    sum(self.embedding_dimension      )
						    + self.num_dynamic_real_features
						    + self.num_time_features
						    + self.num_static_real_features
						    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
						)




