


'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__snake_case				 =	logging.get_logger(__name__)

__snake_case				 =	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case				 =	[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__snake_case				 =	{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__snake_case				 =	{F"""funnel-transformer/{name}""": 512 for name in _model_names}
__snake_case				 =	{F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}
class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							VOCAB_FILES_NAMES
		_a       					=							PRETRAINED_VOCAB_FILES_MAP
		_a       					=							PRETRAINED_INIT_CONFIGURATION
		_a       					=							FunnelTokenizer
		_a       					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a       					=							2




		def __init__(						self       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=True       ,  UpperCamelCase_="<unk>"       ,  UpperCamelCase_="<sep>"       ,  UpperCamelCase_="<pad>"       ,  UpperCamelCase_="<cls>"       ,  UpperCamelCase_="<mask>"       ,  UpperCamelCase_="<s>"       ,  UpperCamelCase_="</s>"       ,  UpperCamelCase_=True       ,  UpperCamelCase_=True       ,  UpperCamelCase_=None       ,  UpperCamelCase_="##"       ,  **UpperCamelCase_       ,  ):

							'''simple docstring'''


							super().__init__(
							    UpperCamelCase_       ,  tokenizer_file=UpperCamelCase_       ,  do_lower_case=UpperCamelCase_       ,  unk_token=UpperCamelCase_       ,  sep_token=UpperCamelCase_       ,  pad_token=UpperCamelCase_       ,  cls_token=UpperCamelCase_       ,  mask_token=UpperCamelCase_       ,  bos_token=UpperCamelCase_       ,  eos_token=UpperCamelCase_       ,  clean_text=UpperCamelCase_       ,  tokenize_chinese_chars=UpperCamelCase_       ,  strip_accents=UpperCamelCase_       ,  wordpieces_prefix=UpperCamelCase_       ,  **UpperCamelCase_       ,  )

							UpperCamelCase__  :int				   =					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
							if (
							    normalizer_state.get('''lowercase'''       ,  UpperCamelCase_				) != do_lower_case
							    or normalizer_state.get('''strip_accents'''       ,  UpperCamelCase_				) != strip_accents
							    or normalizer_state.get('''handle_chinese_chars'''       ,  UpperCamelCase_				) != tokenize_chinese_chars
							):
												UpperCamelCase__  :int				   =					getattr(UpperCamelCase_       ,  normalizer_state.pop('''type'''				)				)
												UpperCamelCase__  :int				   =					do_lower_case
												UpperCamelCase__  :List[str]				   =					strip_accents
												UpperCamelCase__  :Optional[Any]				   =					tokenize_chinese_chars
												UpperCamelCase__  :Dict				   =					normalizer_class(**UpperCamelCase_				)

							UpperCamelCase__  :Dict				   =					do_lower_case




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_=None				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							UpperCamelCase__  :List[str]				   =					[self.sep_token_id]
							UpperCamelCase__  :Dict				   =					[self.cls_token_id]
							if token_ids_a is None:
												return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
							return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							UpperCamelCase__  :List[str]				   =					self._tokenizer.model.save(UpperCamelCase_       ,  name=UpperCamelCase_				)
							return tuple(UpperCamelCase_				)



'''simple docstring'''
import importlib
import os
import sys


# This is required to make the module import works (when the python process is running from the root of the repo)
sys.path.append('''.''')


def      a (     __a  )    ->      List[str]:



					'''simple docstring'''






					UpperCamelCase__  :int				   =					test_file.split(os.path.sep  )
					if components[0:2] != ["tests", "models"]:
										raise ValueError(
										    '''`test_file` should start with `tests/models/` (with `/` being the OS specific path separator). Got '''
										    f'''{test_file} instead.'''  )
					UpperCamelCase__  :List[Any]				   =					components[-1]
					if not test_fn.endswith('''py'''  ):
										raise ValueError(f'''`test_file` should be a python file. Got {test_fn} instead.'''  )
					if not test_fn.startswith('''test_modeling_'''  ):
										raise ValueError(
										    f'''`test_file` should point to a file name of the form `test_modeling_*.py`. Got {test_fn} instead.'''  )

					UpperCamelCase__  :Optional[Any]				   =					components[:-1] + [test_fn.replace('''.py'''					,   ''''''  )]
					UpperCamelCase__  :str				   =					'''.'''.join(__a  )

					return test_module_path


def      a (     __a  )    ->      Tuple:



					'''simple docstring'''






					UpperCamelCase__  :Dict				   =					get_module_path(__a  )
					UpperCamelCase__  :List[str]				   =					importlib.import_module(__a  )

					return test_module


def      a (     __a  )    ->      int:



					'''simple docstring'''






					UpperCamelCase__  :Dict				   =					[]
					UpperCamelCase__  :Optional[Any]				   =					get_test_module(__a  )
					for attr in dir(__a  ):
										if attr.endswith('''ModelTester'''  ):
															tester_classes.append(getattr(__a					,   __a  )  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      List[Any]:



					'''simple docstring'''






					UpperCamelCase__  :List[Any]				   =					[]
					UpperCamelCase__  :Dict				   =					get_test_module(__a  )
					for attr in dir(__a  ):
										UpperCamelCase__  :Dict				   =					getattr(__a					,   __a  )
										# (TF/Flax)ModelTesterMixin is also an attribute in specific model test module. Let's exclude them by checking
										# `all_model_classes` is not empty (which also excludes other special classes).
										UpperCamelCase__  :Optional[int]				   =					getattr(__a					,   '''all_model_classes'''					,   []  )
										if len(__a  ) > 0:
															test_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__  :int				   =					get_test_classes(__a  )
					UpperCamelCase__  :Any				   =					set()
					for test_class in test_classes:
										model_classes.update(test_class.all_model_classes  )

					# sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      Dict:



					'''simple docstring'''






					UpperCamelCase__  :Optional[Any]				   =					test_class()
					if hasattr(__a					,   '''setUp'''  ):
										test.setUp()

					UpperCamelCase__  :Union[str, Any]				   =					None
					if hasattr(__a					,   '''model_tester'''  ):
										# `(TF/Flax)ModelTesterMixin` has this attribute default to `None`. Let's skip this case.
										if test.model_tester is not None:
															UpperCamelCase__  :List[str]				   =					test.model_tester.__class__

					return model_tester


def      a (     __a					,   __a  )    ->      Any:



					'''simple docstring'''






					UpperCamelCase__  :str				   =					get_test_classes(__a  )

					UpperCamelCase__  :List[str]				   =					[]
					for test_class in test_classes:
										if model_class in test_class.all_model_classes:
															target_test_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a					,   __a  )    ->      Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__  :Optional[int]				   =					get_test_classes_for_model(__a					,   __a  )

					UpperCamelCase__  :List[str]				   =					[]
					for test_class in test_classes:
										UpperCamelCase__  :int				   =					get_model_tester_from_test_class(__a  )
										if tester_class is not None:
															tester_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      int:



					'''simple docstring'''






					UpperCamelCase__  :Any				   =					get_test_classes(__a  )
					UpperCamelCase__  :Dict				   =					{test_class: get_model_tester_from_test_class(__a  ) for test_class in test_classes}
					return test_tester_mapping


def      a (     __a  )    ->      List[str]:



					'''simple docstring'''






					UpperCamelCase__  :Any				   =					get_model_classes(__a  )
					UpperCamelCase__  :Tuple				   =					{
					    model_class: get_test_classes_for_model(__a					,   __a  ) for model_class in model_classes
					}
					return model_test_mapping


def      a (     __a  )    ->      Tuple:



					'''simple docstring'''






					UpperCamelCase__  :List[str]				   =					get_model_classes(__a  )
					UpperCamelCase__  :Union[str, Any]				   =					{
					    model_class: get_tester_classes_for_model(__a					,   __a  ) for model_class in model_classes
					}
					return model_to_tester_mapping







def      a (     __a  )    ->      List[Any]:



					'''simple docstring'''






					if isinstance(__a					,   __a  ):
										return o
					elif isinstance(__a					,   __a  ):
										return o.__name__
					elif isinstance(__a					,   (list, tuple)  ):
										return [to_json(__a  ) for x in o]
					elif isinstance(__a					,   __a  ):
										return {to_json(__a  ): to_json(__a  ) for k, v in o.items()}
					else:
										return o