
import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs




class        _SCREAMING_SNAKE_CASE	( snake_case_		):



	def __init__(       self		,    lowercase		,    lowercase=None		,    lowercase=True		,    lowercase=None		,    **lowercase					)							->			Tuple:
					lowerCamelCase_         =     parent
					lowerCamelCase_         =     config_class
					lowerCamelCase_         =     has_text_modality
					lowerCamelCase_         =     kwargs
					lowerCamelCase_         =     common_properties



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict					)
					lowerCamelCase_         =     (
					    ["hidden_size", "num_attention_heads", "num_hidden_layers"]
					    if self.common_properties is None
					    else self.common_properties
					)

					# Add common fields for text models
					if self.has_text_modality:
									common_properties.extend(["vocab_size"]					)

					# Test that config has the common properties as getters
					for prop in common_properties:
									self.parent.assertTrue(hasattr(lowercase		,    lowercase					)		,    msg=f'`{prop}` does not exist'					)

					# Test that config has the common properties as setter
					for idx, name in enumerate(lowercase					):
									try:
													setattr(lowercase		,    lowercase		,    lowercase					)
													self.parent.assertEqual(
													    getattr(lowercase		,    lowercase					)		,    lowercase		,    msg=f'`{name} value {idx} expected, but was {getattr(lowercase		,    lowercase					)}'					)
									except NotImplementedError:
													# Some models might not be able to implement setters for common_properties
													# In that case, a NotImplementedError is raised
													pass

        # Test if config class can be called with Config(prop_name=..)
					for idx, name in enumerate(lowercase					):
									try:
													lowerCamelCase_         =     self.config_class(**{name: idx}					)
													self.parent.assertEqual(
													    getattr(lowercase		,    lowercase					)		,    lowercase		,    msg=f'`{name} value {idx} expected, but was {getattr(lowercase		,    lowercase					)}'					)
									except NotImplementedError:
													# Some models might not be able to implement setters for common_properties
													# In that case, a NotImplementedError is raised
													pass



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict					)
					lowerCamelCase_         =     json.loads(config.to_json_string()					)
					for key, value in self.inputs_dict.items():
									self.parent.assertEqual(obj[key]		,    lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict					)

					with tempfile.TemporaryDirectory() as tmpdirname:
									lowerCamelCase_         =     os.path.join(lowercase		,    "config.json"					)
									config_first.to_json_file(lowercase					)
									lowerCamelCase_         =     self.config_class.from_json_file(lowercase					)

					self.parent.assertEqual(config_second.to_dict()		,    config_first.to_dict()					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Tuple:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict					)

					with tempfile.TemporaryDirectory() as tmpdirname:
									config_first.save_pretrained(lowercase					)
									lowerCamelCase_         =     self.config_class.from_pretrained(lowercase					)

					self.parent.assertEqual(config_second.to_dict()		,    config_first.to_dict()					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[Any]:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict					)

					lowerCamelCase_         =     "test"
					with tempfile.TemporaryDirectory() as tmpdirname:
									lowerCamelCase_         =     os.path.join(lowercase		,    lowercase					)
									config_first.save_pretrained(lowercase					)
									lowerCamelCase_         =     self.config_class.from_pretrained(lowercase		,    subfolder=lowercase					)

					self.parent.assertEqual(config_second.to_dict()		,    config_first.to_dict()					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					lowerCamelCase_         =     self.config_class(**self.inputs_dict		,    num_labels=5					)
					self.parent.assertEqual(len(config.idalabel					)		,    5					)
					self.parent.assertEqual(len(config.labelaid					)		,    5					)

					lowerCamelCase_         =     3
					self.parent.assertEqual(len(config.idalabel					)		,    3					)
					self.parent.assertEqual(len(config.labelaid					)		,    3					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			str:
					if self.config_class.is_composition:
									return
					lowerCamelCase_         =     self.config_class()
					self.parent.assertIsNotNone(lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					lowerCamelCase_         =     copy.deepcopy(lowercase					)
					lowerCamelCase_         =     self.config_class(**lowercase					)
					lowerCamelCase_         =     []
					for key, value in config_common_kwargs.items():
									if key == "torch_dtype":
													if not is_torch_available():
																	continue
													else:
																	import torch

																	if config.torch_dtype != torch.floataa:
																					wrong_values.append(("torch_dtype", config.torch_dtype, torch.floataa)					)
									elif getattr(lowercase		,    lowercase					) != value:
													wrong_values.append((key, getattr(lowercase		,    lowercase					), value)					)

					if len(lowercase					) > 0:
									lowerCamelCase_         =     "\n".join([f'- {v[0]}: got {v[1]} instead of {v[2]}' for v in wrong_values]					)
									raise ValueError(f'The following keys were not properly set in the config:\n{errors}'					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					self.create_and_test_config_common_properties()
					self.create_and_test_config_to_json_string()
					self.create_and_test_config_to_json_file()
					self.create_and_test_config_from_and_save_pretrained()
					self.create_and_test_config_from_and_save_pretrained_subfolder()
					self.create_and_test_config_with_num_labels()
					self.check_config_can_be_init_without_params()
					self.check_config_arguments_init()




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A							=logging.get_logger(__name__)

__A							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A							=[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A							={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A							={F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__A							={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					VOCAB_FILES_NAMES
	lowerCAmelCase__   =					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__   =					PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__   =					FunnelTokenizer
	lowerCAmelCase__   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__   =					2



	def __init__(       self		,    lowercase=None		,    lowercase=None		,    lowercase=True		,    lowercase="<unk>"		,    lowercase="<sep>"		,    lowercase="<pad>"		,    lowercase="<cls>"		,    lowercase="<mask>"		,    lowercase="<s>"		,    lowercase="</s>"		,    lowercase=True		,    lowercase=True		,    lowercase=None		,    lowercase="##"		,    **lowercase		,    )							->			List[str]:
					super().__init__(
					    lowercase		,    tokenizer_file=lowercase		,    do_lower_case=lowercase		,    unk_token=lowercase		,    sep_token=lowercase		,    pad_token=lowercase		,    cls_token=lowercase		,    mask_token=lowercase		,    bos_token=lowercase		,    eos_token=lowercase		,    clean_text=lowercase		,    tokenize_chinese_chars=lowercase		,    strip_accents=lowercase		,    wordpieces_prefix=lowercase		,    **lowercase		,    )

					lowerCamelCase_         =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
					if (
					    normalizer_state.get("lowercase"		,    lowercase					) != do_lower_case
					    or normalizer_state.get("strip_accents"		,    lowercase					) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,    lowercase					) != tokenize_chinese_chars
					):
									lowerCamelCase_         =     getattr(lowercase		,    normalizer_state.pop("type"					)					)
									lowerCamelCase_         =     do_lower_case
									lowerCamelCase_         =     strip_accents
									lowerCamelCase_         =     tokenize_chinese_chars
									lowerCamelCase_         =     normalizer_class(**lowercase					)

					lowerCamelCase_         =     do_lower_case



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase=None					)							->			str:
					lowerCamelCase_         =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			List[int]:
					lowerCamelCase_         =     [self.sep_token_id]
					lowerCamelCase_         =     [self.cls_token_id]
					if token_ids_a is None:
									return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
					return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			Tuple[str]:
					lowerCamelCase_         =     self._tokenizer.model.save(lowercase		,    name=lowercase					)
					return tuple(lowercase					)


