






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase__				:str				   =       logging.get_logger(__name__)

lowercase__				:List[Any]				   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__				:List[str]				   =       [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowercase__				:List[Any]				   =       {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowercase__				:Optional[Any]				   =       {f"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
lowercase__				:Any				   =       {f"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       int         =       VOCAB_FILES_NAMES
      _A     :       Dict         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Optional[int]         =       PRETRAINED_INIT_CONFIGURATION
      _A     :       Union[str, Any]         =       FunnelTokenizer
      _A     :       int         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A     :       int         =       2

      def __init__(							self       :      int						,				__lowercase       :      Union[str, Any]=None						,				__lowercase       :      int=None						,				__lowercase       :      List[str]=True						,				__lowercase       :      Union[str, Any]="<unk>"						,				__lowercase       :      List[Any]="<sep>"						,				__lowercase       :      str="<pad>"						,				__lowercase       :      Dict="<cls>"						,				__lowercase       :      List[str]="<mask>"						,				__lowercase       :      List[Any]="<s>"						,				__lowercase       :      Union[str, Any]="</s>"						,				__lowercase       :      Optional[int]=True						,				__lowercase       :      List[Any]=True						,				__lowercase       :      Tuple=None						,				__lowercase       :      Optional[Any]="##"						,				**__lowercase       :      str						,				):


        '''simple docstring'''




        super().__init__(
            __lowercase						,				tokenizer_file=__lowercase						,				do_lower_case=__lowercase						,				unk_token=__lowercase						,				sep_token=__lowercase						,				pad_token=__lowercase						,				cls_token=__lowercase						,				mask_token=__lowercase						,				bos_token=__lowercase						,				eos_token=__lowercase						,				clean_text=__lowercase						,				tokenize_chinese_chars=__lowercase						,				strip_accents=__lowercase						,				wordpieces_prefix=__lowercase						,				**__lowercase						,				)

        __UpperCAmelCase    :						str       =						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('''lowercase'''						,				__lowercase ) != do_lower_case
            or normalizer_state.get('''strip_accents'''						,				__lowercase ) != strip_accents
            or normalizer_state.get('''handle_chinese_chars'''						,				__lowercase ) != tokenize_chinese_chars
        ):
          __UpperCAmelCase    :						Optional[int]       =						getattr(__lowercase						,				normalizer_state.pop('''type''' ) )
          __UpperCAmelCase    :						Optional[Any]       =						do_lower_case
          __UpperCAmelCase    :						str       =						strip_accents
          __UpperCAmelCase    :						Optional[Any]       =						tokenize_chinese_chars
          __UpperCAmelCase    :						Any       =						normalizer_class(**__lowercase )

        __UpperCAmelCase    :						Dict       =						do_lower_case

      def       A_					(							self       :      List[str]						,				__lowercase       :      Tuple						,				__lowercase       :      Dict=None ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output

      def       A_					(							self       :      Any						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						[self.sep_token_id]
        __UpperCAmelCase    :						int       =						[self.cls_token_id]
        if token_ids_a is None:
          return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
        return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







      def       A_					(							self       :      Tuple						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						self._tokenizer.model.save(__lowercase						,				name=__lowercase )
        return tuple(__lowercase )






"""simple docstring"""





# flake8: noqa
# Lint as: python3
lowercase__				:Tuple				   =       [
    'VerificationMode',
    'Version',
    'disable_progress_bar',
    'enable_progress_bar',
    'is_progress_bar_enabled',
    'experimental',
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental