
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__lowerCAmelCase					   =       logging.get_logger(__name__)

__lowerCAmelCase					   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase					   =       [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__lowerCAmelCase					   =       {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__lowerCAmelCase					   =       {F'''funnel-transformer/{name}''': 5_1_2 for name in _model_names}
__lowerCAmelCase					   =       {F'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}





class     lowerCamelCase      (       __lowerCamelCase					):
	UpperCamelCase_  :					Any    		=	VOCAB_FILES_NAMES
	UpperCamelCase_  :					List[str]    		=	PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase_  :					int    		=	PRETRAINED_INIT_CONFIGURATION
	UpperCamelCase_  :					Optional[int]    		=	FunnelTokenizer
	UpperCamelCase_  :					List[Any]    		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase_  :					int    		=	2







	def __init__(     self      :Union[str, Any]   ,     lowercase      :List[Any]=None   ,     lowercase      :int=None   ,     lowercase      :Any=True   ,     lowercase      :str="<unk>"   ,     lowercase      :Any="<sep>"   ,     lowercase      :str="<pad>"   ,     lowercase      :Union[str, Any]="<cls>"   ,     lowercase      :Optional[int]="<mask>"   ,     lowercase      :Tuple="<s>"   ,     lowercase      :str="</s>"   ,     lowercase      :Any=True   ,     lowercase      :Optional[Any]=True   ,     lowercase      :Any=None   ,     lowercase      :Tuple="##"   ,     **lowercase      :Dict   ,     )		->      List[Any]:







				"""simple docstring"""






				super().__init__(
				    lowercase   ,     tokenizer_file=lowercase   ,     do_lower_case=lowercase   ,     unk_token=lowercase   ,     sep_token=lowercase   ,     pad_token=lowercase   ,     cls_token=lowercase   ,     mask_token=lowercase   ,     bos_token=lowercase   ,     eos_token=lowercase   ,     clean_text=lowercase   ,     tokenize_chinese_chars=lowercase   ,     strip_accents=lowercase   ,     wordpieces_prefix=lowercase   ,     **lowercase   ,     )

				SCREAMING_SNAKE_CASE     						= json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
				if (
				    normalizer_state.get('''lowercase'''   ,     lowercase			) != do_lower_case
				    or normalizer_state.get('''strip_accents'''   ,     lowercase			) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''   ,     lowercase			) != tokenize_chinese_chars
				):
							SCREAMING_SNAKE_CASE     						= getattr(lowercase   ,     normalizer_state.pop('''type'''			)			)
							SCREAMING_SNAKE_CASE     						= do_lower_case
							SCREAMING_SNAKE_CASE     						= strip_accents
							SCREAMING_SNAKE_CASE     						= tokenize_chinese_chars
							SCREAMING_SNAKE_CASE     						= normalizer_class(**lowercase			)

				SCREAMING_SNAKE_CASE     						= do_lower_case







	def  snake_case__     (     self      :Any   ,     lowercase      :Any   ,     lowercase      :Tuple=None			)		->      int:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output







	def  snake_case__     (     self      :Optional[Any]   ,     lowercase      :List[int]   ,     lowercase      :Optional[List[int]] = None			)		->      List[int]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= [self.sep_token_id]
				SCREAMING_SNAKE_CASE     						= [self.cls_token_id]
				if token_ids_a is None:
							return len(cls			) * [self.cls_token_type_id] + len(token_ids_a + sep			) * [0]
				return len(cls			) * [self.cls_token_type_id] + len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







	def  snake_case__     (     self      :List[Any]   ,     lowercase      :str   ,     lowercase      :Optional[str] = None			)		->      Tuple[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE     						= self._tokenizer.model.save(lowercase   ,     name=lowercase			)
				return tuple(lowercase			)

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCAmelCase					   =       {
    'configuration_bloom': ['BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BloomConfig', 'BloomOnnxConfig'],
}
try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase					   =       ['BloomTokenizerFast']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase					   =       [
							    'BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'BloomForCausalLM',
							    'BloomModel',
							    'BloomPreTrainedModel',
							    'BloomForSequenceClassification',
							    'BloomForTokenClassification',
							    'BloomForQuestionAnswering',
							]

if TYPE_CHECKING:
							from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_bloom_fast import BloomTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_bloom import (
														    BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    BloomForCausalLM,
														    BloomForQuestionAnswering,
														    BloomForSequenceClassification,
														    BloomForTokenClassification,
														    BloomModel,
														    BloomPreTrainedModel,
														)

else:
							import sys

							__lowerCAmelCase					   =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)