






"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase_							      = logging.get_logger(__name__)

lowerCamelCase_							      = {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_							      = [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase_							      = {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase_							      = {f'funnel-transformer/{name}': 512 for name in _model_names}
lowerCamelCase_							      = {f'funnel-transformer/{name}': {'''do_lower_case''': True} for name in _model_names}







class      UpperCamelCase_			(__A	):
	__magic_name__       =							VOCAB_FILES_NAMES
	__magic_name__       =							PRETRAINED_VOCAB_FILES_MAP
	__magic_name__       =							PRETRAINED_INIT_CONFIGURATION
	__magic_name__       =							FunnelTokenizer
	__magic_name__       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__       =							2





	def __init__( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					Dict=None      ,      lowerCAmelCase_   :					Optional[int]=None      ,      lowerCAmelCase_   :					Dict=True      ,      lowerCAmelCase_   :					Any="<unk>"      ,      lowerCAmelCase_   :					Tuple="<sep>"      ,      lowerCAmelCase_   :					Tuple="<pad>"      ,      lowerCAmelCase_   :					Optional[Any]="<cls>"      ,      lowerCAmelCase_   :					Tuple="<mask>"      ,      lowerCAmelCase_   :					int="<s>"      ,      lowerCAmelCase_   :					Dict="</s>"      ,      lowerCAmelCase_   :					Union[str, Any]=True      ,      lowerCAmelCase_   :					Dict=True      ,      lowerCAmelCase_   :					int=None      ,      lowerCAmelCase_   :					Union[str, Any]="##"      ,      **lowerCAmelCase_   :					str      ,      )		->     int:
				super().__init__(
				    lowerCAmelCase_      ,      tokenizer_file=lowerCAmelCase_      ,      do_lower_case=lowerCAmelCase_      ,      unk_token=lowerCAmelCase_      ,      sep_token=lowerCAmelCase_      ,      pad_token=lowerCAmelCase_      ,      cls_token=lowerCAmelCase_      ,      mask_token=lowerCAmelCase_      ,      bos_token=lowerCAmelCase_      ,      eos_token=lowerCAmelCase_      ,      clean_text=lowerCAmelCase_      ,      tokenize_chinese_chars=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_      ,      wordpieces_prefix=lowerCAmelCase_      ,      **lowerCAmelCase_      ,      )

				UpperCAmelCase_	:					Tuple     	=   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
				if (
				    normalizer_state.get("lowercase"      ,      lowerCAmelCase_		) != do_lower_case
				    or normalizer_state.get("strip_accents"      ,      lowerCAmelCase_		) != strip_accents
				    or normalizer_state.get("handle_chinese_chars"      ,      lowerCAmelCase_		) != tokenize_chinese_chars
				):
							UpperCAmelCase_	:					str     	=   getattr(lowerCAmelCase_      ,      normalizer_state.pop("type"		)		)
							UpperCAmelCase_	:					List[Any]     	=   do_lower_case
							UpperCAmelCase_	:					List[Any]     	=   strip_accents
							UpperCAmelCase_	:					Dict     	=   tokenize_chinese_chars
							UpperCAmelCase_	:					Optional[Any]     	=   normalizer_class(**lowerCAmelCase_		)

				UpperCAmelCase_	:					Optional[Any]     	=   do_lower_case





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Optional[int]=None		)		->     Optional[Any]:
				UpperCAmelCase_	:					Dict     	=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]      ,      lowerCAmelCase_   :					List[int]      ,      lowerCAmelCase_   :					Optional[List[int]] = None		)		->     List[int]:
				UpperCAmelCase_	:					Tuple     	=   [self.sep_token_id]
				UpperCAmelCase_	:					List[Any]     	=   [self.cls_token_id]
				if token_ids_a is None:
							return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
				return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any      ,      lowerCAmelCase_   :					str      ,      lowerCAmelCase_   :					Optional[str] = None		)		->     Tuple[str]:
				UpperCAmelCase_	:					Union[str, Any]     	=   self._tokenizer.model.save(lowerCAmelCase_      ,      name=lowerCAmelCase_		)
				return tuple(lowerCAmelCase_		)










def       SCREAMING_SNAKE_CASE__  (       _lowercase     : str			)   ->							str:




		'''simple docstring'''
		if not all(char in '01' for char in bin_string			):
				raise ValueError('Non-binary value was passed to the function'			)
		if not bin_string:
				raise ValueError('Empty string was passed to the function'			)
		lowercase__					:      Tuple      =		''
		while len(_lowercase			) % 3 != 0:
				lowercase__					:      Optional[int]      =		'0' + bin_string
		lowercase__					:      List[Any]      =		[
		    bin_string[index : index + 3]
		    for index in range(len(_lowercase			)			)
		    if index % 3 == 0
		]
		for bin_group in bin_string_in_3_list:
				lowercase__					:      int      =		0
				for index, val in enumerate(_lowercase			):
						oct_val += int(2 ** (2 - index) * int(_lowercase			)			)
				oct_string += str(_lowercase			)
		return oct_string


if __name__ == "__main__":
			from doctest import testmod

			testmod()


