





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a     :    Dict         =						logging.get_logger(__name__)

_a     :    Union[str, Any]         =						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a     :    str         =						[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_a     :    Optional[Any]         =						{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_a     :    str         =						{f"funnel-transformer/{name}": 512 for name in _model_names}
_a     :    Tuple         =						{f"funnel-transformer/{name}": {'do_lower_case': True} for name in _model_names}
class   a_ (			A__      ):
			A__      :			List[Any]       	=  VOCAB_FILES_NAMES
			A__      :			Tuple       	=  PRETRAINED_VOCAB_FILES_MAP
			A__      :			Dict       	=  PRETRAINED_INIT_CONFIGURATION
			A__      :			Dict       	=  FunnelTokenizer
			A__      :			List[Any]       	=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__      :			Dict       	=  2

			def __init__(   self   :  Union[str, Any]     , UpperCAmelCase__   :  Optional[Any]=None     , UpperCAmelCase__   :  Any=None     , UpperCAmelCase__   :  List[str]=True     , UpperCAmelCase__   :  Optional[int]="<unk>"     , UpperCAmelCase__   :  Optional[int]="<sep>"     , UpperCAmelCase__   :  Tuple="<pad>"     , UpperCAmelCase__   :  Union[str, Any]="<cls>"     , UpperCAmelCase__   :  Union[str, Any]="<mask>"     , UpperCAmelCase__   :  Optional[int]="<s>"     , UpperCAmelCase__   :  List[Any]="</s>"     , UpperCAmelCase__   :  List[Any]=True     , UpperCAmelCase__   :  List[Any]=True     , UpperCAmelCase__   :  Optional[Any]=None     , UpperCAmelCase__   :  List[str]="##"     , **UpperCAmelCase__   :  Any     , ):






							"""simple docstring"""
							super().__init__(
							    UpperCamelCase_     , tokenizer_file=UpperCamelCase_     , do_lower_case=UpperCamelCase_     , unk_token=UpperCamelCase_     , sep_token=UpperCamelCase_     , pad_token=UpperCamelCase_     , cls_token=UpperCamelCase_     , mask_token=UpperCamelCase_     , bos_token=UpperCamelCase_     , eos_token=UpperCamelCase_     , clean_text=UpperCamelCase_     , tokenize_chinese_chars=UpperCamelCase_     , strip_accents=UpperCamelCase_     , wordpieces_prefix=UpperCamelCase_     , **UpperCamelCase_     , )

							snake_case   :					int    						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
							if (
							    normalizer_state.get('''lowercase'''     , UpperCamelCase_						) != do_lower_case
							    or normalizer_state.get('''strip_accents'''     , UpperCamelCase_						) != strip_accents
							    or normalizer_state.get('''handle_chinese_chars'''     , UpperCamelCase_						) != tokenize_chinese_chars
							):
											snake_case   :					int    						=				getattr(UpperCamelCase_     , normalizer_state.pop('''type'''						)						)
											snake_case   :					int    						=				do_lower_case
											snake_case   :					List[str]    						=				strip_accents
											snake_case   :					Optional[Any]    						=				tokenize_chinese_chars
											snake_case   :					Dict    						=				normalizer_class(**UpperCamelCase_						)

							snake_case   :					Dict    						=				do_lower_case

			def      lowerCAmelCase(   self   :  Optional[Any]     , UpperCAmelCase__   :  Dict     , UpperCAmelCase__   :  Dict=None						):






							"""simple docstring"""
							snake_case   :					Dict    						=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

							return output

			def      lowerCAmelCase(   self   :  Tuple     , UpperCAmelCase__   :  Optional[int]     , UpperCAmelCase__   :  Any = None						):






							"""simple docstring"""
							snake_case   :					List[str]    						=				[self.sep_token_id]
							snake_case   :					Dict    						=				[self.cls_token_id]
							if token_ids_a is None:
											return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
							return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



			def      lowerCAmelCase(   self   :  Dict     , UpperCAmelCase__   :  int     , UpperCAmelCase__   :  List[Any] = None						):






							"""simple docstring"""
							snake_case   :					List[str]    						=				self._tokenizer.model.save(UpperCamelCase_     , name=UpperCamelCase_						)
							return tuple(UpperCamelCase_						)




'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


__snake_case				 =	{
    '''configuration_gpt_neo''': ['''GPT_NEO_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPTNeoConfig''', '''GPTNeoOnnxConfig'''],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case				 =	[
		    '''GPT_NEO_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''GPTNeoForCausalLM''',
		    '''GPTNeoForQuestionAnswering''',
		    '''GPTNeoForSequenceClassification''',
		    '''GPTNeoForTokenClassification''',
		    '''GPTNeoModel''',
		    '''GPTNeoPreTrainedModel''',
		    '''load_tf_weights_in_gpt_neo''',
		]

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case				 =	[
		    '''FlaxGPTNeoForCausalLM''',
		    '''FlaxGPTNeoModel''',
		    '''FlaxGPTNeoPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_gpt_neo import GPT_NEO_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoConfig, GPTNeoOnnxConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_gpt_neo import (
				    GPT_NEO_PRETRAINED_MODEL_ARCHIVE_LIST,
				    GPTNeoForCausalLM,
				    GPTNeoForQuestionAnswering,
				    GPTNeoForSequenceClassification,
				    GPTNeoForTokenClassification,
				    GPTNeoModel,
				    GPTNeoPreTrainedModel,
				    load_tf_weights_in_gpt_neo,
				)

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_gpt_neo import FlaxGPTNeoForCausalLM, FlaxGPTNeoModel, FlaxGPTNeoPreTrainedModel


else:
		import sys

		__snake_case				 =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)