





"""simple docstring"""






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 				__lowercase			(			_UpperCamelCase   ,      unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  BertTokenizer
					__lowerCAmelCase  =  BertTokenizerFast
					__lowerCAmelCase  =  True
					__lowerCAmelCase  =  True
					__lowerCAmelCase  =  filter_non_english







					def  _lowerCamelCase		(						self     ):
									super().setUp()

									__a :						str			       =  [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									__a :						List[str]			       =  os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''vocab_file''']     )
									with open(self.vocab_file	,			'''w'''	,			encoding='''utf-8'''     ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]     )     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Any			       =  '''UNwant\u00E9d,running'''
									__a :						Tuple			       =  '''unwanted, running'''
									return input_text, output_text







					def  _lowerCamelCase		(						self     ):
									__a :						str			       =  self.tokenizer_class(self.vocab_file     )

									__a :						Any			       =  tokenizer.tokenize('''UNwant\u00E9d,running'''     )
									self.assertListEqual(_UpperCAmelCase	,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']     )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase     )	,			[9, 6, 7, 12, 10, 11]     )







					def  _lowerCamelCase		(						self     ):
									if not self.test_rust_tokenizer:
													return

									__a :						Optional[Any]			       =  self.get_tokenizer()
									__a :						Dict			       =  self.get_rust_tokenizer()

									__a :						Union[str, Any]			       =  '''UNwant\u00E9d,running'''

									__a :						Optional[Any]			       =  tokenizer.tokenize(_UpperCAmelCase     )
									__a :						Optional[Any]			       =  rust_tokenizer.tokenize(_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						Union[str, Any]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
									__a :						List[str]			       =  rust_tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						str			       =  self.get_rust_tokenizer()
									__a :						str			       =  tokenizer.encode(_UpperCAmelCase     )
									__a :						Any			       =  rust_tokenizer.encode(_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									# With lower casing
									__a :						Dict			       =  self.get_tokenizer(do_lower_case=_UpperCAmelCase     )
									__a :						List[str]			       =  self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase     )

									__a :						Optional[Any]			       =  '''UNwant\u00E9d,running'''

									__a :						str			       =  tokenizer.tokenize(_UpperCAmelCase     )
									__a :						List[str]			       =  rust_tokenizer.tokenize(_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						Tuple			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
									__a :						Union[str, Any]			       =  rust_tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						Optional[Any]			       =  self.get_rust_tokenizer()
									__a :						Optional[Any]			       =  tokenizer.encode(_UpperCAmelCase     )
									__a :						Tuple			       =  rust_tokenizer.encode(_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''     )	,			['''ah''', '''\u535A''', '''\u63A8''', '''zz''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''     )	,			['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )	,			['''hello''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[Any]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase	,			strip_accents=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )	,			['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )	,			['''h\u00E9llo''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						Dict			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase	,			strip_accents=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )	,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )	,			['''hello''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[str]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )	,			['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']     )
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''     )	,			['''hello''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						int			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''     )	,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[str]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase	,			strip_accents=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )	,			['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						Union[str, Any]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase	,			strip_accents=_UpperCAmelCase     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''     )	,			['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[str]			       =  BasicTokenizer(do_lower_case=_UpperCAmelCase	,			never_split=['''[UNK]''']     )

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''     )	,			['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[Any]			       =  BasicTokenizer()
									__a :						Optional[Any]			       =  '''a\n\'ll !!to?\'d of, can\'t.'''
									__a :						List[Any]			       =  ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
									self.assertListEqual(tokenizer.tokenize(_UpperCAmelCase     )	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						int			       =  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									__a :						List[str]			       =  {}
									for i, token in enumerate(_UpperCAmelCase     ):
													__a :						str			       =  i
									__a :						Optional[int]			       =  WordpieceTokenizer(vocab=_UpperCAmelCase	,			unk_token='''[UNK]'''     )

									self.assertListEqual(tokenizer.tokenize(''''''     )	,			[]     )

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''     )	,			['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']     )

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''     )	,			['''[UNK]''', '''runn''', '''##ing''']     )







					def  _lowerCamelCase		(						self     ):
									self.assertTrue(_is_whitespace(''' '''     )     )
									self.assertTrue(_is_whitespace('''\t'''     )     )
									self.assertTrue(_is_whitespace('''\r'''     )     )
									self.assertTrue(_is_whitespace('''\n'''     )     )
									self.assertTrue(_is_whitespace('''\u00A0'''     )     )

									self.assertFalse(_is_whitespace('''A'''     )     )
									self.assertFalse(_is_whitespace('''-'''     )     )







					def  _lowerCamelCase		(						self     ):
									self.assertTrue(_is_control('''\u0005'''     )     )

									self.assertFalse(_is_control('''A'''     )     )
									self.assertFalse(_is_control(''' '''     )     )
									self.assertFalse(_is_control('''\t'''     )     )
									self.assertFalse(_is_control('''\r'''     )     )







					def  _lowerCamelCase		(						self     ):
									self.assertTrue(_is_punctuation('''-'''     )     )
									self.assertTrue(_is_punctuation('''$'''     )     )
									self.assertTrue(_is_punctuation('''`'''     )     )
									self.assertTrue(_is_punctuation('''.'''     )     )

									self.assertFalse(_is_punctuation('''A'''     )     )
									self.assertFalse(_is_punctuation(''' '''     )     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  self.get_tokenizer()
									__a :						List[str]			       =  self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase     ) for t in ['''Test''', '''\xad''', '''test''']]	,			[['''[UNK]'''], [], ['''[UNK]''']]     )

									self.assertListEqual(
									    [rust_tokenizer.tokenize(_UpperCAmelCase     ) for t in ['''Test''', '''\xad''', '''test''']]	,			[['''[UNK]'''], [], ['''[UNK]''']]     )







					@slow
					def  _lowerCamelCase		(						self     ):
									__a :						List[Any]			       =  self.tokenizer_class.from_pretrained('''bert-base-uncased'''     )

									__a :						Optional[int]			       =  tokenizer.encode('''sequence builders'''	,			add_special_tokens=_UpperCAmelCase     )
									__a :						Optional[int]			       =  tokenizer.encode('''multi-sequence build'''	,			add_special_tokens=_UpperCAmelCase     )

									__a :						List[Any]			       =  tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase     )
									__a :						Union[str, Any]			       =  tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase	,			_UpperCAmelCase     )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]







					def  _lowerCamelCase		(						self     ):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																	__a :						List[str]			       =  self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )

																	__a :						Tuple			       =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																	__a :						List[str]			       =  tokenizer_r.encode_plus(
																	    _UpperCAmelCase	,			return_attention_mask=_UpperCAmelCase	,			return_token_type_ids=_UpperCAmelCase	,			return_offsets_mapping=_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase	,			)

																	__a :						Tuple			       =  tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase	,			'''do_lower_case'''     ) else False
																	__a :						Dict			       =  (
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]	,			tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']     )     )
																	self.assertEqual([e[0] for e in expected_results]	,			tokens['''offset_mapping''']     )







					def  _lowerCamelCase		(						self     ):
									__a :						Dict			       =  ['''的''', '''人''', '''有''']
									__a :						Any			       =  ''''''.join(_UpperCAmelCase     )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																	__a :						int			       =  True
																	__a :						List[str]			       =  self.tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )
																	__a :						Optional[int]			       =  self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )

																	__a :						str			       =  tokenizer_p.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	__a :						Union[str, Any]			       =  tokenizer_r.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )

																	__a :						Optional[Any]			       =  tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase     )
																	__a :						Any			       =  tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase     )

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

																	__a :						Optional[Any]			       =  False
																	__a :						Union[str, Any]			       =  self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )
																	__a :						Tuple			       =  self.tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )

																	__a :						int			       =  tokenizer_r.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	__a :						List[str]			       =  tokenizer_p.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )

																	__a :						List[Any]			       =  tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase     )
																	__a :						List[Any]			       =  tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase     )

																	# it is expected that only the first Chinese character is not preceded by "##".
																	__a :						str			       =  [
																	    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_UpperCAmelCase     )
																	]
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )






"""simple docstring"""






from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A 			=  logging.get_logger(__name__)

A 			=  {
    '''huggingface/autoformer-tourism-monthly''': '''https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json''',
}




class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  '''autoformer'''
					__lowerCAmelCase  =  {
					    '''hidden_size''': '''d_model''',
					    '''num_attention_heads''': '''encoder_attention_heads''',
					    '''num_hidden_layers''': '''encoder_layers''',
					}







					def __init__(						self	,			_UpperCAmelCase = None	,			_UpperCAmelCase = None	,			_UpperCAmelCase = "student_t"	,			_UpperCAmelCase = "nll"	,			_UpperCAmelCase = 1	,			_UpperCAmelCase = [1, 2, 3, 4, 5, 6, 7]	,			_UpperCAmelCase = True	,			_UpperCAmelCase = 0	,			_UpperCAmelCase = 0	,			_UpperCAmelCase = 0	,			_UpperCAmelCase = 0	,			_UpperCAmelCase = None	,			_UpperCAmelCase = None	,			_UpperCAmelCase = 64	,			_UpperCAmelCase = 2	,			_UpperCAmelCase = 2	,			_UpperCAmelCase = 2	,			_UpperCAmelCase = 2	,			_UpperCAmelCase = 32	,			_UpperCAmelCase = 32	,			_UpperCAmelCase = "gelu"	,			_UpperCAmelCase = 0.1	,			_UpperCAmelCase = 0.1	,			_UpperCAmelCase = 0.1	,			_UpperCAmelCase = 0.1	,			_UpperCAmelCase = 0.1	,			_UpperCAmelCase = 100	,			_UpperCAmelCase = 0.0_2	,			_UpperCAmelCase = True	,			_UpperCAmelCase=True	,			_UpperCAmelCase = 10	,			_UpperCAmelCase = 25	,			_UpperCAmelCase = 3	,			**_UpperCAmelCase	,			):
									# time series specific configuration
									__a :						Optional[int]			       =  prediction_length
									__a :						Optional[int]			       =  context_length if context_length is not None else prediction_length
									__a :						Union[str, Any]			       =  distribution_output
									__a :						str			       =  loss
									__a :						Optional[Any]			       =  input_size
									__a :						str			       =  num_time_features
									__a :						Optional[Any]			       =  lags_sequence
									__a :						Any			       =  scaling
									__a :						Optional[Any]			       =  num_dynamic_real_features
									__a :						str			       =  num_static_real_features
									__a :						Any			       =  num_static_categorical_features
									if cardinality is not None and num_static_categorical_features > 0:
													if len(_UpperCAmelCase     ) != num_static_categorical_features:
																	raise ValueError(
																	    '''The cardinality should be a list of the same length as `num_static_categorical_features`'''     )
													__a :						List[str]			       =  cardinality
									else:
													__a :						Any			       =  [0]
									if embedding_dimension is not None and num_static_categorical_features > 0:
													if len(_UpperCAmelCase     ) != num_static_categorical_features:
																	raise ValueError(
																	    '''The embedding dimension should be a list of the same length as `num_static_categorical_features`'''     )
													__a :						Any			       =  embedding_dimension
									else:
													__a :						Tuple			       =  [min(50	,			(cat + 1) // 2     ) for cat in self.cardinality]
									__a :						Union[str, Any]			       =  num_parallel_samples

									# Transformer architecture configuration
									__a :						Union[str, Any]			       =  input_size * len(self.lags_sequence     ) + self._number_of_features
									__a :						List[Any]			       =  d_model
									__a :						List[str]			       =  encoder_attention_heads
									__a :						Union[str, Any]			       =  decoder_attention_heads
									__a :						Optional[int]			       =  encoder_ffn_dim
									__a :						List[str]			       =  decoder_ffn_dim
									__a :						List[str]			       =  encoder_layers
									__a :						Optional[int]			       =  decoder_layers

									__a :						Union[str, Any]			       =  dropout
									__a :						Union[str, Any]			       =  attention_dropout
									__a :						Optional[Any]			       =  activation_dropout
									__a :						List[str]			       =  encoder_layerdrop
									__a :						Any			       =  decoder_layerdrop

									__a :						Union[str, Any]			       =  activation_function
									__a :						Union[str, Any]			       =  init_std

									__a :						List[str]			       =  use_cache

									# Autoformer
									__a :						int			       =  label_length
									__a :						List[Any]			       =  moving_average
									__a :						str			       =  autocorrelation_factor

									super().__init__(is_encoder_decoder=_UpperCAmelCase	,			**_UpperCAmelCase     )







					@property
					def  _lowerCamelCase		(						self     ):
									return (
									    sum(self.embedding_dimension     )
									    + self.num_dynamic_real_features
									    + self.num_time_features
									    + self.num_static_real_features
									    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
									)