


'''simple docstring'''
import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       FunnelTokenizer
		_a									=       FunnelTokenizerFast
		_a									=       True
		_a									=       True




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :Dict							      =				[
							    '''<unk>''',
							    '''<cls>''',
							    '''<sep>''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							UpperCamelCase__     :Optional[int]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)




		def 					lowerCAmelCase__       (	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							return FunnelTokenizer.from_pretrained(self.tmpdirname					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							return FunnelTokenizerFast.from_pretrained(self.tmpdirname					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				'''UNwant\u00E9d,running'''
							UpperCamelCase__     :Union[str, Any]							      =				'''unwanted, running'''
							return input_text, output_text




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.tokenizer_class(self.vocab_file							)

							UpperCamelCase__     :List[Any]							      =				tokenizer.tokenize('''UNwant\u00E9d,running'''							)
							self.assertListEqual(UpperCamelCase_					,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)					,     [7, 4, 5, 10, 8, 9]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												UpperCamelCase__     :Optional[Any]							      =				tokenizer('''UNwant\u00E9d,running'''							)
												UpperCamelCase__     :Tuple							      =				len(inputs['''input_ids''']							) - 1
												self.assertListEqual(inputs['''token_type_ids''']					,     [2] + [0] * sentence_len							)

												UpperCamelCase__     :Union[str, Any]							      =				tokenizer('''UNwant\u00E9d,running'''					,     '''UNwant\u00E9d,running'''							)
												self.assertListEqual(inputs['''token_type_ids''']					,     [2] + [0] * sentence_len + [1] * sentence_len							)



'''simple docstring'''
class      lowercase			:

		"""simple docstring"""

		def __init__(	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				n
							UpperCamelCase__     :Tuple							      =				[None] * self.n
							UpperCamelCase__     :str							      =				0  # index of the first element
							UpperCamelCase__     :List[Any]							      =				0
							UpperCamelCase__     :Dict							      =				0




		def __len__(	self							):

							'''simple docstring'''


							return self.size




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return self.size == 0




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return False if self.is_empty() else self.array[self.front]




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if self.size >= self.n:
												raise Exception('''QUEUE IS FULL'''							)

							UpperCamelCase__     :List[Any]							      =				data
							UpperCamelCase__     :List[Any]							      =				(self.rear + 1) % self.n
							self.size += 1
							return self




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							if self.size == 0:
												raise Exception('''UNDERFLOW'''							)

							UpperCamelCase__     :Dict							      =				self.array[self.front]
							UpperCamelCase__     :Union[str, Any]							      =				None
							UpperCamelCase__     :Union[str, Any]							      =				(self.front + 1) % self.n
							self.size -= 1
							return temp