




"""simple docstring"""




import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class   UpperCAmelCase_    (			snake_case   ,					unittest.TestCase		):
		UpperCamelCase							=FunnelTokenizer
		UpperCamelCase							=FunnelTokenizerFast
		UpperCamelCase							=True
		UpperCamelCase							=True
		def 	_lowerCamelCase							(						self       )						->       List[Any]:
					super().setUp()

					__lowercase			:  int								=       [
					    '''<unk>''',
					    '''<cls>''',
					    '''<sep>''',
					    '''want''',
					    '''##want''',
					    '''##ed''',
					    '''wa''',
					    '''un''',
					    '''runn''',
					    '''##ing''',
					    ''',''',
					    '''low''',
					    '''lowest''',
					]
					__lowercase			:  int								=       os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['''vocab_file''']       )
					with open(self.vocab_file				,						'''w'''				,						encoding='''utf-8'''       ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )
		def 	_lowerCamelCase							(						self				,						**UpperCamelCase_       )						->       int:
					return FunnelTokenizer.from_pretrained(self.tmpdirname				,						**UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						**UpperCamelCase_       )						->       str:
					return FunnelTokenizerFast.from_pretrained(self.tmpdirname				,						**UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       str:
					__lowercase			:  Dict								=       '''UNwant\u00E9d,running'''
					__lowercase			:  List[str]								=       '''unwanted, running'''
					return input_text, output_text
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					__lowercase			:  int								=       self.tokenizer_class(self.vocab_file       )

					__lowercase			:  Union[str, Any]								=       tokenizer.tokenize('''UNwant\u00E9d,running'''       )
					self.assertListEqual(UpperCamelCase_				,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_       )				,						[7, 4, 5, 10, 8, 9]       )
		def 	_lowerCamelCase							(						self       )						->       Union[str, Any]:
					__lowercase			:  str								=       self.get_tokenizers(do_lower_case=UpperCamelCase_       )
					for tokenizer in tokenizers:
								__lowercase			:  Union[str, Any]								=       tokenizer('''UNwant\u00E9d,running'''       )
								__lowercase			:  Dict								=       len(inputs['''input_ids''']       ) - 1
								self.assertListEqual(inputs['''token_type_ids''']				,						[2] + [0] * sentence_len       )

								__lowercase			:  Tuple								=       tokenizer('''UNwant\u00E9d,running'''				,						'''UNwant\u00E9d,running'''       )
								self.assertListEqual(inputs['''token_type_ids''']				,						[2] + [0] * sentence_len + [1] * sentence_len       )







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase   : List[Any]             =				{
    """configuration_albert""": ["""ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """AlbertConfig""", """AlbertOnnxConfig"""],
}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase   : Any             =				["""AlbertTokenizer"""]

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase   : Optional[Any]             =				["""AlbertTokenizerFast"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase   : List[str]             =				[
					    """ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """AlbertForMaskedLM""",
					    """AlbertForMultipleChoice""",
					    """AlbertForPreTraining""",
					    """AlbertForQuestionAnswering""",
					    """AlbertForSequenceClassification""",
					    """AlbertForTokenClassification""",
					    """AlbertModel""",
					    """AlbertPreTrainedModel""",
					    """load_tf_weights_in_albert""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase   : Tuple             =				[
					    """TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFAlbertForMaskedLM""",
					    """TFAlbertForMultipleChoice""",
					    """TFAlbertForPreTraining""",
					    """TFAlbertForQuestionAnswering""",
					    """TFAlbertForSequenceClassification""",
					    """TFAlbertForTokenClassification""",
					    """TFAlbertMainLayer""",
					    """TFAlbertModel""",
					    """TFAlbertPreTrainedModel""",
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase   : Optional[int]             =				[
					    """FlaxAlbertForMaskedLM""",
					    """FlaxAlbertForMultipleChoice""",
					    """FlaxAlbertForPreTraining""",
					    """FlaxAlbertForQuestionAnswering""",
					    """FlaxAlbertForSequenceClassification""",
					    """FlaxAlbertForTokenClassification""",
					    """FlaxAlbertModel""",
					    """FlaxAlbertPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_albert import AlbertTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_albert_fast import AlbertTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_albert import (
										    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    AlbertForMaskedLM,
										    AlbertForMultipleChoice,
										    AlbertForPreTraining,
										    AlbertForQuestionAnswering,
										    AlbertForSequenceClassification,
										    AlbertForTokenClassification,
										    AlbertModel,
										    AlbertPreTrainedModel,
										    load_tf_weights_in_albert,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_albert import (
										    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFAlbertForMaskedLM,
										    TFAlbertForMultipleChoice,
										    TFAlbertForPreTraining,
										    TFAlbertForQuestionAnswering,
										    TFAlbertForSequenceClassification,
										    TFAlbertForTokenClassification,
										    TFAlbertMainLayer,
										    TFAlbertModel,
										    TFAlbertPreTrainedModel,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_albert import (
										    FlaxAlbertForMaskedLM,
										    FlaxAlbertForMultipleChoice,
										    FlaxAlbertForPreTraining,
										    FlaxAlbertForQuestionAnswering,
										    FlaxAlbertForSequenceClassification,
										    FlaxAlbertForTokenClassification,
										    FlaxAlbertModel,
										    FlaxAlbertPreTrainedModel,
										)
else:
					import sys

					lowerCAmelCase   : Union[str, Any]             =				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


