






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A					=	logging.get_logger(__name__)

__A					=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A					=	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

__A					=	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
__A					=	{F"""funnel-transformer/{name}""": 5_12 for name in _model_names}
__A					=	{F"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}



class 				_A			(     UpperCamelCase				):
    """simple docstring"""

    lowerCamelCase	:							Optional[Any]				= VOCAB_FILES_NAMES
    lowerCamelCase	:							Optional[int]				= PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase	:							Union[str, Any]				= PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase	:							Tuple				= FunnelTokenizer
    lowerCamelCase	:							Optional[int]				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase	:							int				= 2



    def __init__(				self      :     List[Any] ,     __SCREAMING_SNAKE_CASE      :     str=None ,     __SCREAMING_SNAKE_CASE      :     Any=None ,     __SCREAMING_SNAKE_CASE      :     int=True ,     __SCREAMING_SNAKE_CASE      :     Optional[int]="<unk>" ,     __SCREAMING_SNAKE_CASE      :     Any="<sep>" ,     __SCREAMING_SNAKE_CASE      :     Optional[int]="<pad>" ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]="<cls>" ,     __SCREAMING_SNAKE_CASE      :     List[str]="<mask>" ,     __SCREAMING_SNAKE_CASE      :     Dict="<s>" ,     __SCREAMING_SNAKE_CASE      :     Any="</s>" ,     __SCREAMING_SNAKE_CASE      :     str=True ,     __SCREAMING_SNAKE_CASE      :     Dict=True ,     __SCREAMING_SNAKE_CASE      :     Optional[int]=None ,     __SCREAMING_SNAKE_CASE      :     Dict="##" ,     **__SCREAMING_SNAKE_CASE      :     Tuple ,     )      ->							List[Any]:
           super().__init__(
               __SCREAMING_SNAKE_CASE ,     tokenizer_file=__SCREAMING_SNAKE_CASE ,     do_lower_case=__SCREAMING_SNAKE_CASE ,     unk_token=__SCREAMING_SNAKE_CASE ,     sep_token=__SCREAMING_SNAKE_CASE ,     pad_token=__SCREAMING_SNAKE_CASE ,     cls_token=__SCREAMING_SNAKE_CASE ,     mask_token=__SCREAMING_SNAKE_CASE ,     bos_token=__SCREAMING_SNAKE_CASE ,     eos_token=__SCREAMING_SNAKE_CASE ,     clean_text=__SCREAMING_SNAKE_CASE ,     tokenize_chinese_chars=__SCREAMING_SNAKE_CASE ,     strip_accents=__SCREAMING_SNAKE_CASE ,     wordpieces_prefix=__SCREAMING_SNAKE_CASE ,     **__SCREAMING_SNAKE_CASE ,     )

           __UpperCAmelCase     				=json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
           if (
               normalizer_state.get("""lowercase""" ,     __SCREAMING_SNAKE_CASE      ) != do_lower_case
               or normalizer_state.get("""strip_accents""" ,     __SCREAMING_SNAKE_CASE      ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars""" ,     __SCREAMING_SNAKE_CASE      ) != tokenize_chinese_chars
           ):
                  __UpperCAmelCase     				=getattr(__SCREAMING_SNAKE_CASE ,     normalizer_state.pop("""type"""      )      )
                  __UpperCAmelCase     				=do_lower_case
                  __UpperCAmelCase     				=strip_accents
                  __UpperCAmelCase     				=tokenize_chinese_chars
                  __UpperCAmelCase     				=normalizer_class(**__SCREAMING_SNAKE_CASE      )

           __UpperCAmelCase     				=do_lower_case



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     List[Any] ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=None      )      ->							Any:
           __UpperCAmelCase     				=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output



    def _a							(				self      :     str ,     __SCREAMING_SNAKE_CASE      :     List[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[List[int]] = None      )      ->							List[int]:
           __UpperCAmelCase     				=[self.sep_token_id]
           __UpperCAmelCase     				=[self.cls_token_id]
           if token_ids_a is None:
                  return len(cls      ) * [self.cls_token_type_id] + len(token_ids_a + sep      ) * [0]
           return len(cls      ) * [self.cls_token_type_id] + len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



    def _a							(				self      :     Any ,     __SCREAMING_SNAKE_CASE      :     str ,     __SCREAMING_SNAKE_CASE      :     Optional[str] = None      )      ->							Tuple[str]:
           __UpperCAmelCase     				=self._tokenizer.model.save(__SCREAMING_SNAKE_CASE ,     name=__SCREAMING_SNAKE_CASE      )
           return tuple(__SCREAMING_SNAKE_CASE      )








from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A				:       Optional[Any]      =  {'configuration_van': ['VAN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VanConfig']}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A				:       Optional[Any]      =  [
	    'VAN_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'VanForImageClassification',
	    'VanModel',
	    'VanPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_van import VAN_PRETRAINED_CONFIG_ARCHIVE_MAP, VanConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_van import (
		    VAN_PRETRAINED_MODEL_ARCHIVE_LIST,
		    VanForImageClassification,
		    VanModel,
		    VanPreTrainedModel,
		)

else:
	import sys

	A				:       Union[str, Any]      =  _LazyModule(__name__, globals()['__file__'], _import_structure)





