


from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case    :   Optional[int]	    =					logging.get_logger(__name__)

snake_case    :   Dict	    =					{
    '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/config.json''',
    '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/config.json''',
    '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/config.json''',
    '''funnel-transformer/medium-base''': '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/config.json''',
    '''funnel-transformer/intermediate''': (
        '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/config.json'''
    ),
    '''funnel-transformer/intermediate-base''': (
        '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/config.json'''
    ),
    '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/config.json''',
    '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/config.json''',
    '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/config.json''',
    '''funnel-transformer/xlarge-base''': '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/config.json''',
}







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	'funnel'
 SCREAMING_SNAKE_CASE__				=	{
     'hidden_size': 'd_model',
     'num_attention_heads': 'n_head',
 }




 def __init__(		self ,  _lowerCamelCase=3_0522 ,  _lowerCamelCase=[4, 4, 4] ,  _lowerCamelCase=None ,  _lowerCamelCase=2 ,  _lowerCamelCase=768 ,  _lowerCamelCase=12 ,  _lowerCamelCase=64 ,  _lowerCamelCase=3072 ,  _lowerCamelCase="gelu_new" ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=0.0 ,  _lowerCamelCase=0.1 ,  _lowerCamelCase=None ,  _lowerCamelCase=1e-9 ,  _lowerCamelCase="mean" ,  _lowerCamelCase="relative_shift" ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  _lowerCamelCase=True ,  **_lowerCamelCase ,  ):
       a						:Union[str, Any]										=	vocab_size
       a						:Any										=	block_sizes
       a						:List[Any]										=	[1] * len(_lowerCamelCase   ) if block_repeats is None else block_repeats
       assert len(_lowerCamelCase   ) == len(
           self.block_repeats   ), "`block_sizes` and `block_repeats` should have the same length."
       a						:Optional[int]										=	num_decoder_layers
       a						:List[Any]										=	d_model
       a						:Dict										=	n_head
       a						:Tuple										=	d_head
       a						:List[str]										=	d_inner
       a						:List[Any]										=	hidden_act
       a						:List[str]										=	hidden_dropout
       a						:List[Any]										=	attention_dropout
       a						:Optional[int]										=	activation_dropout
       a						:List[Any]										=	initializer_range
       a						:Optional[Any]										=	initializer_std
       a						:Optional[int]										=	layer_norm_eps
       assert pooling_type in [
           "mean",
           "max",
       ], F'''Got {pooling_type} for `pooling_type` but only \'mean\' and \'max\' are supported.'''
       a						:List[str]										=	pooling_type
       assert attention_type in [
           "relative_shift",
           "factorized",
       ], F'''Got {attention_type} for `attention_type` but only \'relative_shift\' and \'factorized\' are supported.'''
       a						:List[str]										=	attention_type
       a						:Union[str, Any]										=	separate_cls
       a						:Any										=	truncate_seq
       a						:Optional[int]										=	pool_q_only

       super().__init__(**_lowerCamelCase   )




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return sum(self.block_sizes   )




 @num_hidden_layers.setter
 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       raise NotImplementedError(
           '''This model does not support the setting of `num_hidden_layers`. Please set `block_sizes`.'''   )




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return len(self.block_sizes   )




 @num_blocks.setter
 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       raise NotImplementedError('''This model does not support the setting of `num_blocks`. Please set `block_sizes`.'''   )









import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


snake_case    :   List[Any]	    =					get_tests_dir('''fixtures/test_sentencepiece_bpe.model''')







class 		_snake_case  (				_snake_case						,   unittest.TestCase							):
 SCREAMING_SNAKE_CASE__				=	BartphoTokenizer
 SCREAMING_SNAKE_CASE__				=	False
 SCREAMING_SNAKE_CASE__				=	True




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       super().setUp()

       a						:Dict										=	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
       a						:Optional[Any]										=	dict(zip(_lowerCamelCase ,  range(len(_lowerCamelCase   )   )   )   )
       a						:Tuple										=	{'''unk_token''': '''<unk>'''}

       a						:Union[str, Any]										=	os.path.join(self.tmpdirname ,  VOCAB_FILES_NAMES['''monolingual_vocab_file''']   )
       with open(self.monolingual_vocab_file ,  '''w''' ,  encoding='''utf-8'''   ) as fp:
             for token in vocab_tokens:
                   fp.write(F'''{token} {vocab_tokens[token]}\n'''   )

       a						:Any										=	BartphoTokenizer(_lowerCamelCase ,  self.monolingual_vocab_file ,  **self.special_tokens_map   )
       tokenizer.save_pretrained(self.tmpdirname   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  **_lowerCamelCase   ):
       kwargs.update(self.special_tokens_map   )
       return BartphoTokenizer.from_pretrained(self.tmpdirname ,  **_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:int										=	'''This is a là test'''
       a						:str										=	'''This is a<unk><unk> test'''
       return input_text, output_text




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       a						:List[str]										=	BartphoTokenizer(_lowerCamelCase ,  self.monolingual_vocab_file ,  **self.special_tokens_map   )
       a						:Optional[Any]										=	'''This is a là test'''
       a						:Tuple										=	'''▁This ▁is ▁a ▁l à ▁t est'''.split()
       a						:int										=	tokenizer.tokenize(_lowerCamelCase   )
       self.assertListEqual(_lowerCamelCase ,  _lowerCamelCase   )

       a						:Union[str, Any]										=	tokens + [tokenizer.unk_token]
       a						:str										=	[4, 5, 6, 3, 3, 7, 8, 3]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase   ) ,  _lowerCamelCase   )





