


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_lowercase       :      List[Any]	=	logging.get_logger(__name__)

_lowercase       :      List[str]	=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowercase       :      Tuple	=	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_lowercase       :      List[Any]	=	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_lowercase       :      Any	=	{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_lowercase       :      Optional[Any]	=	{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}






class       _UpperCamelCase    (					__snake_case				):

				"""simple docstring"""







				lowerCAmelCase 		= VOCAB_FILES_NAMES
				lowerCAmelCase 		= PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase 		= PRETRAINED_INIT_CONFIGURATION
				lowerCAmelCase 		= FunnelTokenizer
				lowerCAmelCase 		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase 		= 2

				def __init__(						self				,   a__=None				,   a__=None				,   a__=True				,   a__="<unk>"				,   a__="<sep>"				,   a__="<pad>"				,   a__="<cls>"				,   a__="<mask>"				,   a__="<s>"				,   a__="</s>"				,   a__=True				,   a__=True				,   a__=None				,   a__="##"				,   **a__				,   )	->				Any:
					super().__init__(
					    a__				,   tokenizer_file=a__				,   do_lower_case=a__				,   unk_token=a__				,   sep_token=a__				,   pad_token=a__				,   cls_token=a__				,   mask_token=a__				,   bos_token=a__				,   eos_token=a__				,   clean_text=a__				,   tokenize_chinese_chars=a__				,   strip_accents=a__				,   wordpieces_prefix=a__				,   **a__				,   )

					A								=				json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
					if (
					    normalizer_state.get("""lowercase"""				,   a__   ) != do_lower_case
					    or normalizer_state.get("""strip_accents"""				,   a__   ) != strip_accents
					    or normalizer_state.get("""handle_chinese_chars"""				,   a__   ) != tokenize_chinese_chars
					):
						A								=				getattr(a__				,   normalizer_state.pop("""type"""   )   )
						A								=				do_lower_case
						A								=				strip_accents
						A								=				tokenize_chinese_chars
						A								=				normalizer_class(**a__   )

					A								=				do_lower_case

				def      _UpperCAmelCase					(						self				,   a__				,   a__=None   )	->				Optional[Any]:
					A								=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output

				def      _UpperCAmelCase					(						self				,   a__				,   a__ = None   )	->				List[int]:
					A								=				[self.sep_token_id]
					A								=				[self.cls_token_id]
					if token_ids_a is None:
						return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
					return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

				def      _UpperCAmelCase					(						self				,   a__				,   a__ = None   )	->				Tuple[str]:
					A								=				self._tokenizer.model.save(a__				,   name=a__   )
					return tuple(a__   )







import math





def   _lowerCAmelCase				(   UpperCamelCase__:     int						) ->   bool:





	"""simple docstring"""







	if 1 < number < 4:
		# 2 and 3 are primes
		return True
	elif number < 2 or number % 2 == 0 or number % 3 == 0:
		# Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
		return False

	# All primes number are in format of 6k +/- 1
	for i in range(5							,					int(math.sqrt(UpperCamelCase__						) + 1						)							,					6						):
		if number % i == 0 or number % (i + 2) == 0:
			return False
	return True





def   _lowerCAmelCase				(   UpperCamelCase__:     float = 0.1						) ->   int:





	"""simple docstring"""







	A								=				3
	A								=				3

	while primes / (2 * j - 1) >= ratio:
		for i in range(j * j + j + 1							,					(j + 2) * (j + 2)							,					j + 1						):
			primes += is_prime(UpperCamelCase__						)
		j += 2
	return j


if __name__ == "__main__":
				import doctest

				doctest.testmod()



