





"""simple docstring"""





import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    _lowerCAmelCase  (	a    ,  unittest.TestCase			):
       """simple docstring"""




       __magic_name__	:List[Any]						    =   FunnelTokenizer
       __magic_name__	:Union[str, Any]						    =   FunnelTokenizerFast
       __magic_name__	:Tuple						    =   True
       __magic_name__	:Optional[Any]						    =   True






       def 	snake_case		(       self ):



         '''simple docstring'''

         super().setUp()

         lowerCAmelCase__     :int											=				[
             '<unk>',
             '<cls>',
             '<sep>',
             'want',
             '##want',
             '##ed',
             'wa',
             'un',
             'runn',
             '##ing',
             ',',
             'low',
             'lowest',
         ]
         lowerCAmelCase__     :Union[str, Any]											=				os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES['vocab_file'] )
         with open(self.vocab_file       ,					'w'       ,					encoding='utf-8' ) as vocab_writer:
           vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )






       def 	snake_case		(       self       ,					**__UpperCAmelCase ):



         '''simple docstring'''

         return FunnelTokenizer.from_pretrained(self.tmpdirname       ,					**__UpperCAmelCase )






       def 	snake_case		(       self       ,					**__UpperCAmelCase ):



         '''simple docstring'''

         return FunnelTokenizerFast.from_pretrained(self.tmpdirname       ,					**__UpperCAmelCase )






       def 	snake_case		(       self       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[int]											=				'UNwant\u00E9d,running'
         lowerCAmelCase__     :List[str]											=				'unwanted, running'
         return input_text, output_text






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Dict											=				self.tokenizer_class(self.vocab_file )

         lowerCAmelCase__     :int											=				tokenizer.tokenize('UNwant\u00E9d,running' )
         self.assertListEqual(__UpperCAmelCase       ,					['un', '##want', '##ed', ',', 'runn', '##ing'] )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase )       ,					[7, 4, 5, 1_0, 8, 9] )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__     :Any											=				self.get_tokenizers(do_lower_case=__UpperCAmelCase )
         for tokenizer in tokenizers:
           lowerCAmelCase__     :Optional[int]											=				tokenizer('UNwant\u00E9d,running' )
           lowerCAmelCase__     :Any											=				len(inputs['input_ids'] ) - 1
           self.assertListEqual(inputs['token_type_ids']       ,					[2] + [0] * sentence_len )

           lowerCAmelCase__     :int											=				tokenizer('UNwant\u00E9d,running'       ,					'UNwant\u00E9d,running' )
           self.assertListEqual(inputs['token_type_ids']       ,					[2] + [0] * sentence_len + [1] * sentence_len )




"""simple docstring"""




__magic_name__						: Optional[Any]    =				'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'





def       a_ (	lowercase__ :bytes     ):
			# Make sure the supplied data is a bytes-like object
			if not isinstance(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      f'a bytes-like object is required, not \'{data.__class__.__name__}\''
						raise TypeError(lowercase__     )

			__lowerCamelCase          =      """""".join(bin(lowercase__     )[2:].zfill(8     ) for byte in data     )

			__lowerCamelCase          =      len(lowercase__     ) % 6 != 0

			if padding_needed:
						# The padding that will be added later
						__lowerCamelCase          =      B"""=""" * ((6 - len(lowercase__     ) % 6) // 2)

						# Append binary_stream with arbitrary binary digits (0's by default) to make its
						# length a multiple of 6.
						binary_stream += "0" * (6 - len(lowercase__     ) % 6)
			else:
						__lowerCamelCase          =      B""""""

			# Encode every 6 binary digits to their corresponding Base64 character
			return (
			    "".join(
			        B64_CHARSET[int(binary_stream[index : index + 6],			2     )]
			        for index in range(0,			len(lowercase__     ),			6     )     ).encode()
			    + padding
			)





def       a_ (	lowercase__ :str     ):
			# Make sure encoded_data is either a string or a bytes-like object
			if not isinstance(lowercase__,			lowercase__     ) and not isinstance(lowercase__,			lowercase__     ):
						__lowerCamelCase          =      (
						    """argument should be a bytes-like object or ASCII string, """
						    f'not \'{encoded_data.__class__.__name__}\''
						)
						raise TypeError(lowercase__     )

			# In case encoded_data is a bytes-like object, make sure it contains only
			# ASCII characters so we convert it to a string object
			if isinstance(lowercase__,			lowercase__     ):
						try:
									__lowerCamelCase          =      encoded_data.decode("""utf-8"""     )
						except UnicodeDecodeError:
									raise ValueError("""base64 encoded data should only contain ASCII characters"""     )

			__lowerCamelCase          =      encoded_data.count("""="""     )

			# Check if the encoded string contains non base64 characters
			if padding:
						assert all(
						    char in B64_CHARSET for char in encoded_data[:-padding]     ), "Invalid base64 character(s) found."
			else:
						assert all(
						    char in B64_CHARSET for char in encoded_data     ), "Invalid base64 character(s) found."

			# Check the padding
			assert len(lowercase__     ) % 4 == 0 and padding < 3, "Incorrect padding"

			if padding:
						# Remove padding if there is one
						__lowerCamelCase          =      encoded_data[:-padding]

						__lowerCamelCase          =      """""".join(
						    bin(B64_CHARSET.index(lowercase__     )     )[2:].zfill(6     ) for char in encoded_data     )[: -padding * 2]
			else:
						__lowerCamelCase          =      """""".join(
						    bin(B64_CHARSET.index(lowercase__     )     )[2:].zfill(6     ) for char in encoded_data     )

			__lowerCamelCase          =      [
			    int(binary_stream[index : index + 8],			2     )
			    for index in range(0,			len(lowercase__     ),			8     )
			]

			return bytes(lowercase__     )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




