
from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
			from transformers import (
			    AutoConfig,
			    BertConfig,
			    GPTaConfig,
			    TaConfig,
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
			from transformers import (
			    AutoModel,
			    AutoModelForCausalLM,
			    AutoModelForMaskedLM,
			    AutoModelForPreTraining,
			    AutoModelForQuestionAnswering,
			    AutoModelForSeqaSeqLM,
			    AutoModelForSequenceClassification,
			    AutoModelWithLMHead,
			    BertForMaskedLM,
			    BertForPreTraining,
			    BertForQuestionAnswering,
			    BertForSequenceClassification,
			    BertModel,
			    GPTaLMHeadModel,
			    RobertaForMaskedLM,
			    TaForConditionalGeneration,
			)




@is_pt_tf_cross_test
class        _SCREAMING_SNAKE_CASE	( unittest.TestCase		):



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModel.from_pretrained(lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModel.from_pretrained(lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForPreTraining.from_pretrained(lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForPreTraining.from_pretrained(lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForCausalLM.from_pretrained(lowercase		,    from_pt=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     TFAutoModelForCausalLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForCausalLM.from_pretrained(lowercase		,    from_tf=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     AutoModelForCausalLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelWithLMHead.from_pretrained(lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelWithLMHead.from_pretrained(lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForMaskedLM.from_pretrained(lowercase		,    from_pt=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     TFAutoModelForMaskedLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForMaskedLM.from_pretrained(lowercase		,    from_tf=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     AutoModelForMaskedLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			List[str]:
					for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForSeqaSeqLM.from_pretrained(lowercase		,    from_pt=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     TFAutoModelForSeqaSeqLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForSeqaSeqLM.from_pretrained(lowercase		,    from_tf=lowercase					)
									lowerCamelCase_      ,       lowerCamelCase_         =     AutoModelForSeqaSeqLM.from_pretrained(
									    lowercase		,    output_loading_info=lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForSequenceClassification.from_pretrained(lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForSequenceClassification.from_pretrained(lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
									lowerCamelCase_         =     AutoConfig.from_pretrained(lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     TFAutoModelForQuestionAnswering.from_pretrained(lowercase		,    from_pt=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)

									lowerCamelCase_         =     AutoModelForQuestionAnswering.from_pretrained(lowercase		,    from_tf=lowercase					)
									self.assertIsNotNone(lowercase					)
									self.assertIsInstance(lowercase		,    lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[int]:
					lowerCamelCase_         =     TFAutoModelWithLMHead.from_pretrained(lowercase		,    from_pt=lowercase					)
					self.assertIsInstance(lowercase		,    lowercase					)
					self.assertEqual(model.num_parameters()		,    14410					)
					self.assertEqual(model.num_parameters(only_trainable=lowercase					)		,    14410					)

					lowerCamelCase_         =     AutoModelWithLMHead.from_pretrained(lowercase		,    from_tf=lowercase					)
					self.assertIsInstance(lowercase		,    lowercase					)
					self.assertEqual(model.num_parameters()		,    14410					)
					self.assertEqual(model.num_parameters(only_trainable=lowercase					)		,    14410					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Tuple:
					lowerCamelCase_         =     TFAutoModelWithLMHead.from_pretrained(lowercase		,    from_pt=lowercase					)
					self.assertIsInstance(lowercase		,    lowercase					)
					self.assertEqual(model.num_parameters()		,    14410					)
					self.assertEqual(model.num_parameters(only_trainable=lowercase					)		,    14410					)

					lowerCamelCase_         =     AutoModelWithLMHead.from_pretrained(lowercase		,    from_tf=lowercase					)
					self.assertIsInstance(lowercase		,    lowercase					)
					self.assertEqual(model.num_parameters()		,    14410					)
					self.assertEqual(model.num_parameters(only_trainable=lowercase					)		,    14410					)




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__A							=logging.get_logger(__name__)

__A							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A							=[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

__A							={
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
__A							={F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
__A							={F"""funnel-transformer/{name}""": {'''do_lower_case''': True} for name in _model_names}




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					VOCAB_FILES_NAMES
	lowerCAmelCase__   =					PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__   =					PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__   =					FunnelTokenizer
	lowerCAmelCase__   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__   =					2



	def __init__(       self		,    lowercase=None		,    lowercase=None		,    lowercase=True		,    lowercase="<unk>"		,    lowercase="<sep>"		,    lowercase="<pad>"		,    lowercase="<cls>"		,    lowercase="<mask>"		,    lowercase="<s>"		,    lowercase="</s>"		,    lowercase=True		,    lowercase=True		,    lowercase=None		,    lowercase="##"		,    **lowercase		,    )							->			List[str]:
					super().__init__(
					    lowercase		,    tokenizer_file=lowercase		,    do_lower_case=lowercase		,    unk_token=lowercase		,    sep_token=lowercase		,    pad_token=lowercase		,    cls_token=lowercase		,    mask_token=lowercase		,    bos_token=lowercase		,    eos_token=lowercase		,    clean_text=lowercase		,    tokenize_chinese_chars=lowercase		,    strip_accents=lowercase		,    wordpieces_prefix=lowercase		,    **lowercase		,    )

					lowerCamelCase_         =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
					if (
					    normalizer_state.get("lowercase"		,    lowercase					) != do_lower_case
					    or normalizer_state.get("strip_accents"		,    lowercase					) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,    lowercase					) != tokenize_chinese_chars
					):
									lowerCamelCase_         =     getattr(lowercase		,    normalizer_state.pop("type"					)					)
									lowerCamelCase_         =     do_lower_case
									lowerCamelCase_         =     strip_accents
									lowerCamelCase_         =     tokenize_chinese_chars
									lowerCamelCase_         =     normalizer_class(**lowercase					)

					lowerCamelCase_         =     do_lower_case



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase=None					)							->			str:
					lowerCamelCase_         =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

					return output



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			List[int]:
					lowerCamelCase_         =     [self.sep_token_id]
					lowerCamelCase_         =     [self.cls_token_id]
					if token_ids_a is None:
									return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
					return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase = None					)							->			Tuple[str]:
					lowerCamelCase_         =     self._tokenizer.model.save(lowercase		,    name=lowercase					)
					return tuple(lowercase					)


