





"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import (
           TFFunnelBaseModel,
           TFFunnelForMaskedLM,
           TFFunnelForMultipleChoice,
           TFFunnelForPreTraining,
           TFFunnelForQuestionAnswering,
           TFFunnelForSequenceClassification,
           TFFunnelForTokenClassification,
           TFFunnelModel,
       )






class 						__snake_case   :





 """simple docstring"""


 def __init__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase=13   ,		__lowerCamelCase=7   ,		__lowerCamelCase=True   ,		__lowerCamelCase=True   ,		__lowerCamelCase=True   ,		__lowerCamelCase=True   ,		__lowerCamelCase=99   ,		__lowerCamelCase=[1, 1, 2]   ,		__lowerCamelCase=1   ,		__lowerCamelCase=32   ,		__lowerCamelCase=4   ,		__lowerCamelCase=8   ,		__lowerCamelCase=37   ,		__lowerCamelCase="gelu_new"   ,		__lowerCamelCase=0.1   ,		__lowerCamelCase=0.1   ,		__lowerCamelCase=0.0   ,		__lowerCamelCase=512   ,		__lowerCamelCase=3   ,		__lowerCamelCase=0.0_2   ,		__lowerCamelCase=3   ,		__lowerCamelCase=4   ,		__lowerCamelCase=None   ,		__lowerCamelCase=False   ,		):



     '''simple docstring'''




     __A       :		List[Any]							=	parent
     __A       :		Optional[int]							=	batch_size
     __A       :		Union[str, Any]							=	seq_length
     __A       :		Any							=	is_training
     __A       :		str							=	use_input_mask
     __A       :		Tuple							=	use_token_type_ids
     __A       :		int							=	use_labels
     __A       :		Optional[Any]							=	vocab_size
     __A       :		int							=	block_sizes
     __A       :		List[str]							=	num_decoder_layers
     __A       :		Dict							=	d_model
     __A       :		List[Any]							=	n_head
     __A       :		Dict							=	d_head
     __A       :		int							=	d_inner
     __A       :		Optional[int]							=	hidden_act
     __A       :		List[str]							=	hidden_dropout
     __A       :		int							=	attention_dropout
     __A       :		Dict							=	activation_dropout
     __A       :		List[Any]							=	max_position_embeddings
     __A       :		Union[str, Any]							=	type_vocab_size
     __A       :		Any							=	2
     __A       :		Any							=	num_labels
     __A       :		Tuple							=	num_choices
     __A       :		Optional[int]							=	scope
     __A       :		str							=	initializer_std

     # Used in the tests to check the size of the first attention layer
     __A       :		Tuple							=	n_head
     # Used in the tests to check the size of the first hidden state
     __A       :		Union[str, Any]							=	self.d_model
     # Used in the tests to check the number of output hidden states/attentions
     __A       :		Tuple							=	sum(self.block_sizes    ) + (0 if base else self.num_decoder_layers)
     # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
     # the last hidden state of the first block (which is the first hidden state of the decoder).
     if not base:
         __A       :		Dict							=	self.num_hidden_layers + 2

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Any							=	ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size    )

     __A       :		List[str]							=	None
     if self.use_input_mask:
         __A       :		Optional[Any]							=	random_attention_mask([self.batch_size, self.seq_length]    )

     __A       :		Dict							=	None
     if self.use_token_type_ids:
         __A       :		Union[str, Any]							=	ids_tensor([self.batch_size, self.seq_length]   ,		self.type_vocab_size    )

     __A       :		List[Any]							=	None
     __A       :		List[str]							=	None
     __A       :		str							=	None
     if self.use_labels:
         __A       :		str							=	ids_tensor([self.batch_size]   ,		self.type_sequence_label_size    )
         __A       :		str							=	ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels    )
         __A       :		int							=	ids_tensor([self.batch_size]   ,		self.num_choices    )

     __A       :		Optional[int]							=	FunnelConfig(
         vocab_size=self.vocab_size   ,		block_sizes=self.block_sizes   ,		num_decoder_layers=self.num_decoder_layers   ,		d_model=self.d_model   ,		n_head=self.n_head   ,		d_head=self.d_head   ,		d_inner=self.d_inner   ,		hidden_act=self.hidden_act   ,		hidden_dropout=self.hidden_dropout   ,		attention_dropout=self.attention_dropout   ,		activation_dropout=self.activation_dropout   ,		max_position_embeddings=self.max_position_embeddings   ,		type_vocab_size=self.type_vocab_size   ,		initializer_std=self.initializer_std   ,		)

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
     )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Union[str, Any]							=	TFFunnelModel(config=_lowerCamelCase    )
     __A       :		List[Any]							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		Optional[int]							=	model(_lowerCamelCase    )

     __A       :		Dict							=	[input_ids, input_mask]
     __A       :		Dict							=	model(_lowerCamelCase    )

     __A       :		Union[str, Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.d_model)    )

     __A       :		Optional[Any]							=	False
     __A       :		Dict							=	TFFunnelModel(config=_lowerCamelCase    )
     __A       :		Dict							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.d_model)    )

     __A       :		Dict							=	False
     __A       :		Optional[Any]							=	TFFunnelModel(config=_lowerCamelCase    )
     __A       :		int							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.d_model)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Optional[int]							=	TFFunnelBaseModel(config=_lowerCamelCase    )
     __A       :		Optional[int]							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		Union[str, Any]							=	model(_lowerCamelCase    )

     __A       :		str							=	[input_ids, input_mask]
     __A       :		Union[str, Any]							=	model(_lowerCamelCase    )

     __A       :		Tuple							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, 2, self.d_model)    )

     __A       :		List[Any]							=	False
     __A       :		Union[str, Any]							=	TFFunnelBaseModel(config=_lowerCamelCase    )
     __A       :		Optional[Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, 3, self.d_model)    )

     __A       :		List[Any]							=	False
     __A       :		Tuple							=	TFFunnelBaseModel(config=_lowerCamelCase    )
     __A       :		List[Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, 2, self.d_model)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		List[Any]							=	TFFunnelForPreTraining(config=_lowerCamelCase    )
     __A       :		Optional[int]							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		Dict							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Dict							=	TFFunnelForMaskedLM(config=_lowerCamelCase    )
     __A       :		str							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		str							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Union[str, Any]							=	self.num_labels
     __A       :		List[Any]							=	TFFunnelForSequenceClassification(config=_lowerCamelCase    )
     __A       :		Tuple							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		List[Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_labels)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		Optional[Any]							=	self.num_choices
     __A       :		int							=	TFFunnelForMultipleChoice(config=_lowerCamelCase    )
     __A       :		Optional[Any]							=	tf.tile(tf.expand_dims(_lowerCamelCase   ,		1    )   ,		(1, self.num_choices, 1)    )
     __A       :		Optional[int]							=	tf.tile(tf.expand_dims(_lowerCamelCase   ,		1    )   ,		(1, self.num_choices, 1)    )
     __A       :		Tuple							=	tf.tile(tf.expand_dims(_lowerCamelCase   ,		1    )   ,		(1, self.num_choices, 1)    )
     __A       :		int							=	{
         '''input_ids''': multiple_choice_inputs_ids,
         '''attention_mask''': multiple_choice_input_mask,
         '''token_type_ids''': multiple_choice_token_type_ids,
     }
     __A       :		Optional[Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_choices)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		List[str]							=	self.num_labels
     __A       :		Optional[Any]							=	TFFunnelForTokenClassification(config=_lowerCamelCase    )
     __A       :		List[str]							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		str							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.num_labels)    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		):



     '''simple docstring'''




     __A       :		int							=	TFFunnelForQuestionAnswering(config=_lowerCamelCase    )
     __A       :		str							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
     __A       :		Optional[Any]							=	model(_lowerCamelCase    )
     self.parent.assertEqual(result.start_logits.shape   ,		(self.batch_size, self.seq_length)    )
     self.parent.assertEqual(result.end_logits.shape   ,		(self.batch_size, self.seq_length)    )







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.prepare_config_and_inputs()
     (
         (
         __A 
     )					,			(
         __A 
     )					,			(
         __A 
     )					,			(
         __A 
     )					,			(
         __A 
     )					,			(
         __A 
     )					,			(
         __A 
     )					,			 
     )       :		List[str]							=	config_and_inputs
     __A       :		Optional[Any]							=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict






@require_tf
class 						__snake_case   (						__lowerCAmelCase				,				__lowerCAmelCase				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	(
     (
         TFFunnelModel,
         TFFunnelForMaskedLM,
         TFFunnelForPreTraining,
         TFFunnelForQuestionAnswering,
         TFFunnelForTokenClassification,
     )
     if is_tf_available()
     else ()
 )
 _lowerCamelCase								=	(
     {
         '''feature-extraction''': (TFFunnelBaseModel, TFFunnelModel),
         '''fill-mask''': TFFunnelForMaskedLM,
         '''question-answering''': TFFunnelForQuestionAnswering,
         '''text-classification''': TFFunnelForSequenceClassification,
         '''token-classification''': TFFunnelForTokenClassification,
         '''zero-shot''': TFFunnelForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 _lowerCamelCase								=	False
 _lowerCamelCase								=	False

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	TFFunnelModelTester(self    )
     __A       :		Union[str, Any]							=	ConfigTester(self   ,		config_class=_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     self.config_tester.run_common_tests()

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Any							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase    )







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase    )






@require_tf
class 						__snake_case   (						__lowerCAmelCase				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	(
     (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
 )
 _lowerCamelCase								=	False
 _lowerCamelCase								=	False

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	TFFunnelModelTester(self   ,		base=_lowerCamelCase    )
     __A       :		Union[str, Any]							=	ConfigTester(self   ,		config_class=_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     self.config_tester.run_common_tests()

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_base_model(*_lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase    )







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Dict							=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase    )





"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_albert import AlbertTokenizer
else:
	a				=None

a				=logging.get_logger(__name__)
a				={'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a				={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

a				={
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

a				='▁'






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  Dict    				=			VOCAB_FILES_NAMES
				A__ :  List[Any]    				=			PRETRAINED_VOCAB_FILES_MAP
				A__ :  List[Any]    				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__ :  Union[str, Any]    				=			AlbertTokenizer

				def __init__( self      ,					_lowerCamelCase=None      ,					_lowerCamelCase=None      ,					_lowerCamelCase=True      ,					_lowerCamelCase=True      ,					_lowerCamelCase=False      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<unk>"      ,					_lowerCamelCase="[SEP]"      ,					_lowerCamelCase="<pad>"      ,					_lowerCamelCase="[CLS]"      ,					_lowerCamelCase="[MASK]"      ,					**_lowerCamelCase      ,					):
										# Mask token behave like a normal word, i.e. include the space before it and
										# is included in the raw text, there should be a match in a non-normalized sentence.
										lowerCamelCase__   =(
										    AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase      ,					normalized=_lowerCamelCase )
										    if isinstance(_lowerCamelCase      ,					_lowerCamelCase )
										    else mask_token
										)

										super().__init__(
										    _lowerCamelCase      ,					tokenizer_file=_lowerCamelCase      ,					do_lower_case=_lowerCamelCase      ,					remove_space=_lowerCamelCase      ,					keep_accents=_lowerCamelCase      ,					bos_token=_lowerCamelCase      ,					eos_token=_lowerCamelCase      ,					unk_token=_lowerCamelCase      ,					sep_token=_lowerCamelCase      ,					pad_token=_lowerCamelCase      ,					cls_token=_lowerCamelCase      ,					mask_token=_lowerCamelCase      ,					**_lowerCamelCase      ,					)

										lowerCamelCase__   =do_lower_case
										lowerCamelCase__   =remove_space
										lowerCamelCase__   =keep_accents
										lowerCamelCase__   =vocab_file
										lowerCamelCase__   =False if not self.vocab_file else True

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.sep_token_id]
										lowerCamelCase__   =[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer." )

										if not os.path.isdir(_lowerCamelCase ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
																return
										lowerCamelCase__   =os.path.join(
										    _lowerCamelCase      ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowerCamelCase ):
																copyfile(self.vocab_file      ,					_lowerCamelCase )

										return (out_vocab_file,)


