
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase    :   Optional[int]						=			logging.get_logger(__name__)

lowercase    :   Dict						=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase    :   int						=			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowercase    :   List[Any]						=			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowercase    :   Tuple						=			{f"funnel-transformer/{name}": 512 for name in _model_names}
lowercase    :   List[str]						=			{f"funnel-transformer/{name}": {'do_lower_case': True} for name in _model_names}




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	_A								=	VOCAB_FILES_NAMES
	_A								=	PRETRAINED_VOCAB_FILES_MAP
	_A								=	PRETRAINED_INIT_CONFIGURATION
	_A								=	FunnelTokenizer
	_A								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A								=	2







	def __init__(   self						:Tuple	,  a						:Optional[int]=None	,  a						:Tuple=None	,  a						:Optional[Any]=True	,  a						:Optional[Any]="<unk>"	,  a						:List[Any]="<sep>"	,  a						:Tuple="<pad>"	,  a						:str="<cls>"	,  a						:Dict="<mask>"	,  a						:Dict="<s>"	,  a						:Any="</s>"	,  a						:Optional[int]=True	,  a						:int=True	,  a						:Dict=None	,  a						:List[str]="##"	,  **a						:Union[str, Any]	,  )						->					Union[str, Any]:
								super().__init__(
								    a	,  tokenizer_file=a	,  do_lower_case=a	,  unk_token=a	,  sep_token=a	,  pad_token=a	,  cls_token=a	,  mask_token=a	,  bos_token=a	,  eos_token=a	,  clean_text=a	,  tokenize_chinese_chars=a	,  strip_accents=a	,  wordpieces_prefix=a	,  **a	,  )

								__UpperCamelCase      :  Union[str, Any]       =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
								if (
								    normalizer_state.get("lowercase"	,  a		) != do_lower_case
								    or normalizer_state.get("strip_accents"	,  a		) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"	,  a		) != tokenize_chinese_chars
								):
															__UpperCamelCase      :  Optional[int]       =   getattr(a	,  normalizer_state.pop("type"		)		)
															__UpperCamelCase      :  Optional[int]       =   do_lower_case
															__UpperCamelCase      :  Dict       =   strip_accents
															__UpperCamelCase      :  Tuple       =   tokenize_chinese_chars
															__UpperCamelCase      :  Tuple       =   normalizer_class(**a		)

								__UpperCamelCase      :  Union[str, Any]       =   do_lower_case







	def   _lowerCamelCase							(   self						:List[Any]	,  a						:List[Any]	,  a						:Optional[int]=None		)						->					List[Any]:
								__UpperCamelCase      :  Optional[Any]       =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

								return output







	def   _lowerCamelCase							(   self						:List[str]	,  a						:List[int]	,  a						:Optional[List[int]] = None		)						->					List[int]:
								__UpperCamelCase      :  str       =   [self.sep_token_id]
								__UpperCamelCase      :  Optional[Any]       =   [self.cls_token_id]
								if token_ids_a is None:
															return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
								return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def   _lowerCamelCase							(   self						:Tuple	,  a						:str	,  a						:Optional[str] = None		)						->					Tuple[str]:
								__UpperCamelCase      :  List[Any]       =   self._tokenizer.model.save(a	,  name=a		)
								return tuple(a		)

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase    :   Tuple						=			{
    'configuration_luke': ['LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LukeConfig'],
    'tokenization_luke': ['LukeTokenizer'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase    :   int						=			[
		    'LUKE_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'LukeForEntityClassification',
		    'LukeForEntityPairClassification',
		    'LukeForEntitySpanClassification',
		    'LukeForMultipleChoice',
		    'LukeForQuestionAnswering',
		    'LukeForSequenceClassification',
		    'LukeForTokenClassification',
		    'LukeForMaskedLM',
		    'LukeModel',
		    'LukePreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_luke import LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP, LukeConfig
		from .tokenization_luke import LukeTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_luke import (
				    LUKE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LukeForEntityClassification,
				    LukeForEntityPairClassification,
				    LukeForEntitySpanClassification,
				    LukeForMaskedLM,
				    LukeForMultipleChoice,
				    LukeForQuestionAnswering,
				    LukeForSequenceClassification,
				    LukeForTokenClassification,
				    LukeModel,
				    LukePreTrainedModel,
				)

else:
		import sys

		lowercase    :   Any						=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)