
import importlib.metadata
import operator
import re
import sys
from typing import Optional

from packaging import version


__A							={
    '''<''': operator.lt,
    '''<=''': operator.le,
    '''==''': operator.eq,
    '''!=''': operator.ne,
    '''>=''': operator.ge,
    '''>''': operator.gt,
}





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__       ):
				if got_ver is None or want_ver is None:
								raise ValueError(
								    F'Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider'
								    F' reinstalling {pkg}.'       )
				if not ops[op](version.parse(lowerCamelCase__       )      ,					version.parse(lowerCamelCase__       )       ):
								raise ImportError(
								    F'{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}'       )





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__ = None       ):
				lowerCamelCase_         =     F'\n{hint}' if hint is not None else ""

				# non-versioned check
				if re.match(r"^[\w_\-\d]+$"      ,					lowerCamelCase__       ):
								lowerCamelCase_      ,       lowerCamelCase_      ,       lowerCamelCase_         =     requirement, None, None
				else:
								lowerCamelCase_         =     re.findall(r"^([^!=<>\s]+)([\s!=<>]{1,2}.+)"      ,					lowerCamelCase__       )
								if not match:
												raise ValueError(
												    "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but"
												    F' got {requirement}'       )
								lowerCamelCase_      ,       lowerCamelCase_         =     match[0]
								lowerCamelCase_         =     want_full.split(","       )  # there could be multiple requirements
								lowerCamelCase_         =     {}
								for w in want_range:
												lowerCamelCase_         =     re.findall(r"^([\s!=<>]{1,2})(.+)"      ,					lowerCamelCase__       )
												if not match:
																raise ValueError(
																    "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,"
																    F' but got {requirement}'       )
												lowerCamelCase_      ,       lowerCamelCase_         =     match[0]
												lowerCamelCase_         =     want_ver
												if op not in ops:
																raise ValueError(F'{requirement}: need one of {list(ops.keys()       )}, but got {op}'       )

    # special case
				if pkg == "python":
								lowerCamelCase_         =     ".".join([str(lowerCamelCase__       ) for x in sys.version_info[:3]]       )
								for op, want_ver in wanted.items():
												_compare_versions(lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__       )
								return

				# check if any version is installed
				try:
								lowerCamelCase_         =     importlib.metadata.version(lowerCamelCase__       )
				except importlib.metadata.PackageNotFoundError:
								raise importlib.metadata.PackageNotFoundError(
								    F'The \'{requirement}\' distribution was not found and is required by this application. {hint}'       )

				# check that the right version is installed if version number or a range was provided
				if want_ver is not None:
								for op, want_ver in wanted.items():
												_compare_versions(lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__       )





def      lowerCamelCase_					(							lowerCamelCase__       ):
				lowerCamelCase_         =     "Try: pip install transformers -U or pip install -e '.[dev]' if you're working with git main"
				return require_version(lowerCamelCase__      ,					lowerCamelCase__       )




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A							=logging.get_logger(__name__)

__A							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A							=[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A							={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A							={F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__A							={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					VOCAB_FILES_NAMES
	lowerCAmelCase__   =					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__   =					PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__   =					FunnelTokenizer
	lowerCAmelCase__   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__   =					2



	def __init__(       self		,    lowercase=None		,    lowercase=None		,    lowercase=True		,    lowercase="<unk>"		,    lowercase="<sep>"		,    lowercase="<pad>"		,    lowercase="<cls>"		,    lowercase="<mask>"		,    lowercase="<s>"		,    lowercase="</s>"		,    lowercase=True		,    lowercase=True		,    lowercase=None		,    lowercase="##"		,    **lowercase		,    )							->			List[str]:
					super().__init__(
					    lowercase		,    tokenizer_file=lowercase		,    do_lower_case=lowercase		,    unk_token=lowercase		,    sep_token=lowercase		,    pad_token=lowercase		,    cls_token=lowercase		,    mask_token=lowercase		,    bos_token=lowercase		,    eos_token=lowercase		,    clean_text=lowercase		,    tokenize_chinese_chars=lowercase		,    strip_accents=lowercase		,    wordpieces_prefix=lowercase		,    **lowercase		,    )

					lowerCamelCase_         =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
					if (
					    normalizer_state.get("lowercase"		,    lowercase					) != do_lower_case
					    or normalizer_state.get("strip_accents"		,    lowercase					) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,    lowercase					) != tokenize_chinese_chars
					):
									lowerCamelCase_         =     getattr(lowercase		,    normalizer_state.pop("type"					)					)
									lowerCamelCase_         =     do_lower_case
									lowerCamelCase_         =     strip_accents
									lowerCamelCase_         =     tokenize_chinese_chars
									lowerCamelCase_         =     normalizer_class(**lowercase					)

					lowerCamelCase_         =     do_lower_case



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase=None					)							->			str:
					lowerCamelCase_         =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			List[int]:
					lowerCamelCase_         =     [self.sep_token_id]
					lowerCamelCase_         =     [self.cls_token_id]
					if token_ids_a is None:
									return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
					return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			Tuple[str]:
					lowerCamelCase_         =     self._tokenizer.model.save(lowercase		,    name=lowercase					)
					return tuple(lowercase					)


