




'''simple docstring'''
import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE			=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE			=						{"vocab_file": "vocab.json"}

_SCREAMING_SNAKE_CASE			=						{
    "vocab_file": {
        "mgp-str": "https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json",
    }
}

_SCREAMING_SNAKE_CASE			=						{"mgp-str": 27}
class   __UpperCAmelCase				(     lowerCAmelCase				):



					'''simple docstring'''





					_UpperCamelCase							=		VOCAB_FILES_NAMES
					_UpperCamelCase							=		PRETRAINED_VOCAB_FILES_MAP
					_UpperCamelCase							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



					def __init__(	self     :						Optional[Any]      ,       _lowercase     :						int      ,       _lowercase     :						Dict="[GO]"      ,       _lowercase     :						Union[str, Any]="[GO]"      ,       _lowercase     :						Union[str, Any]="[s]"      ,       _lowercase     :						Optional[int]="[GO]"      ,       **_lowercase     :						List[str])    ->							Optional[Any]:
								super().__init__(
								    unk_token=_lowercase      ,       bos_token=_lowercase      ,       eos_token=_lowercase      ,       pad_token=_lowercase      ,       **_lowercase      ,       )

								with open(_lowercase      ,       encoding='utf-8') as vocab_handle:
											A_									= json.load(_lowercase)
								A_									= {v: k for k, v in self.vocab.items()}



					@property
					def     __snake_case						(	self     :						Optional[Any])    ->							int:
								return len(self.vocab)



					def     __snake_case						(	self     :						str)    ->							Union[str, Any]:
								return dict(self.vocab      ,       **self.added_tokens_encoder)



					def     __snake_case						(	self     :						int      ,       _lowercase     :						Any)    ->							Optional[Any]:
								A_									= []
								for s in text:
											char_tokens.extend(_lowercase)
								return char_tokens



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						Dict)    ->							Any:
								return self.vocab.get(_lowercase      ,       self.vocab.get(self.unk_token))



					def     __snake_case						(	self     :						int      ,       _lowercase     :						str)    ->							str:
								return self.decoder.get(_lowercase)



					def     __snake_case						(	self     :						Dict      ,       _lowercase     :						str      ,       _lowercase     :						Optional[str] = None)    ->							Tuple[str]:
								if not os.path.isdir(_lowercase):
											logger.error('Vocabulary path ({}) should be a directory'.format(_lowercase))
											return
								A_									= os.path.join(
								    _lowercase      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

								with open(_lowercase      ,       'w'      ,       encoding='utf-8') as f:
											f.write(json.dumps(self.vocab      ,       indent=2      ,       sort_keys=_lowercase      ,       ensure_ascii=_lowercase) + '\n')

								return (vocab_file,)










'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFFunnelBaseModel,
							    TFFunnelForMaskedLM,
							    TFFunnelForMultipleChoice,
							    TFFunnelForPreTraining,
							    TFFunnelForQuestionAnswering,
							    TFFunnelForSequenceClassification,
							    TFFunnelForTokenClassification,
							    TFFunnelModel,
							)
class   __UpperCAmelCase				:



					'''simple docstring'''





					def __init__(	self     :						str      ,       _lowercase     :						List[str]      ,       _lowercase     :						Dict=13      ,       _lowercase     :						List[str]=7      ,       _lowercase     :						Union[str, Any]=True      ,       _lowercase     :						int=True      ,       _lowercase     :						int=True      ,       _lowercase     :						Optional[Any]=True      ,       _lowercase     :						List[Any]=99      ,       _lowercase     :						Optional[Any]=[1, 1, 2]      ,       _lowercase     :						Optional[Any]=1      ,       _lowercase     :						List[str]=32      ,       _lowercase     :						Dict=4      ,       _lowercase     :						List[str]=8      ,       _lowercase     :						List[str]=37      ,       _lowercase     :						int="gelu_new"      ,       _lowercase     :						Optional[int]=0.1      ,       _lowercase     :						List[str]=0.1      ,       _lowercase     :						Union[str, Any]=0.0      ,       _lowercase     :						str=512      ,       _lowercase     :						Optional[Any]=3      ,       _lowercase     :						str=0.02      ,       _lowercase     :						Union[str, Any]=3      ,       _lowercase     :						Optional[Any]=4      ,       _lowercase     :						Tuple=None      ,       _lowercase     :						List[Any]=False      ,       )    ->							List[str]:
								A_									= parent
								A_									= batch_size
								A_									= seq_length
								A_									= is_training
								A_									= use_input_mask
								A_									= use_token_type_ids
								A_									= use_labels
								A_									= vocab_size
								A_									= block_sizes
								A_									= num_decoder_layers
								A_									= d_model
								A_									= n_head
								A_									= d_head
								A_									= d_inner
								A_									= hidden_act
								A_									= hidden_dropout
								A_									= attention_dropout
								A_									= activation_dropout
								A_									= max_position_embeddings
								A_									= type_vocab_size
								A_									= 2
								A_									= num_labels
								A_									= num_choices
								A_									= scope
								A_									= initializer_std

								# Used in the tests to check the size of the first attention layer
								A_									= n_head
								# Used in the tests to check the size of the first hidden state
								A_									= self.d_model
								# Used in the tests to check the number of output hidden states/attentions
								A_									= sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
								# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
								# the last hidden state of the first block (which is the first hidden state of the decoder).
								if not base:
											A_									= self.num_hidden_layers + 2



					def     __snake_case						(	self     :						str)    ->							List[str]:
								A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size)

								A_									= None
								if self.use_input_mask:
											A_									= random_attention_mask([self.batch_size, self.seq_length])

								A_									= None
								if self.use_token_type_ids:
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.type_vocab_size)

								A_									= None
								A_									= None
								A_									= None
								if self.use_labels:
											A_									= ids_tensor([self.batch_size]      ,       self.type_sequence_label_size)
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.num_labels)
											A_									= ids_tensor([self.batch_size]      ,       self.num_choices)

								A_									= FunnelConfig(
								    vocab_size=self.vocab_size      ,       block_sizes=self.block_sizes      ,       num_decoder_layers=self.num_decoder_layers      ,       d_model=self.d_model      ,       n_head=self.n_head      ,       d_head=self.d_head      ,       d_inner=self.d_inner      ,       hidden_act=self.hidden_act      ,       hidden_dropout=self.hidden_dropout      ,       attention_dropout=self.attention_dropout      ,       activation_dropout=self.activation_dropout      ,       max_position_embeddings=self.max_position_embeddings      ,       type_vocab_size=self.type_vocab_size      ,       initializer_std=self.initializer_std      ,       )

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								)



					def     __snake_case						(	self     :						Tuple      ,       _lowercase     :						Tuple      ,       _lowercase     :						str      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						str      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= TFFunnelModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						List[str]      ,       _lowercase     :						Optional[int]      ,       )    ->							Dict:
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 3, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))



					def     __snake_case						(	self     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						Any      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Optional[Any]      ,       )    ->							List[Any]:
								A_									= TFFunnelForPreTraining(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       )    ->							Union[str, Any]:
								A_									= TFFunnelForMaskedLM(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.vocab_size))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Any      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[int]      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= self.num_labels
								A_									= TFFunnelForSequenceClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						str      ,       )    ->							List[str]:
								A_									= self.num_choices
								A_									= TFFunnelForMultipleChoice(config=_lowercase)
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= {
								    'input_ids': multiple_choice_inputs_ids,
								    'attention_mask': multiple_choice_input_mask,
								    'token_type_ids': multiple_choice_token_type_ids,
								}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_choices))



					def     __snake_case						(	self     :						Any      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       )    ->							List[str]:
								A_									= self.num_labels
								A_									= TFFunnelForTokenClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						str      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       )    ->							Optional[int]:
								A_									= TFFunnelForQuestionAnswering(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.start_logits.shape      ,       (self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						str)    ->							Union[str, Any]:
								A_									= self.prepare_config_and_inputs()
								(
								    (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, 
								)									= config_and_inputs
								A_									= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (
					        TFFunnelModel,
					        TFFunnelForMaskedLM,
					        TFFunnelForPreTraining,
					        TFFunnelForQuestionAnswering,
					        TFFunnelForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					_UpperCamelCase							=		(
					    {
					        """feature-extraction""": (TFFunnelBaseModel, TFFunnelModel),
					        """fill-mask""": TFFunnelForMaskedLM,
					        """question-answering""": TFFunnelForQuestionAnswering,
					        """text-classification""": TFFunnelForSequenceClassification,
					        """token-classification""": TFFunnelForTokenClassification,
					        """zero-shot""": TFFunnelForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						Optional[int])    ->							Optional[Any]:
								A_									= TFFunnelModelTester(self)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[Any]:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						Union[str, Any])    ->							Union[str, Any]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_lowercase)



					def     __snake_case						(	self     :						List[Any])    ->							str:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[int]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*_lowercase)



					def     __snake_case						(	self     :						List[str])    ->							Tuple:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*_lowercase)



					def     __snake_case						(	self     :						int)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*_lowercase)





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						str)    ->							Dict:
								A_									= TFFunnelModelTester(self      ,       base=_lowercase)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Optional[Any])    ->							Tuple:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						List[str])    ->							List[str]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_base_model(*_lowercase)



					def     __snake_case						(	self     :						Dict)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*_lowercase)



					def     __snake_case						(	self     :						Union[str, Any])    ->							int:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*_lowercase)




