


'''simple docstring'''


import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class        lowercase__    (		_snake_case	,	unittest.TestCase   ):





							'''simple docstring'''


							A_				: Tuple       =    None
							A_				: Union[str, Any]       =    BloomTokenizerFast
							A_				: Optional[int]       =    BloomTokenizerFast
							A_				: Union[str, Any]       =    True
							A_				: List[Any]       =    False
							A_				: Union[str, Any]       =    """tokenizer_file"""
							A_				: Optional[int]       =    {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






							def UpperCAmelCase_      (       self    ):
											super().setUp()
											_SCREAMING_SNAKE_CASE					:			int							= BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""    )
											tokenizer.save_pretrained(self.tmpdirname    )






							def UpperCAmelCase_      (       self       ,   **__snake_case    ):
											kwargs.update(self.special_tokens_map    )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname       ,   **__snake_case    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= self.get_rust_tokenizer()

											_SCREAMING_SNAKE_CASE					:			int							= ["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
											_SCREAMING_SNAKE_CASE					:			int							= [[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

											_SCREAMING_SNAKE_CASE					:			int							= tokenizer.batch_encode_plus(__snake_case    )["""input_ids"""]
											self.assertListEqual(__snake_case       ,   __snake_case    )

											_SCREAMING_SNAKE_CASE					:			List[str]							= tokenizer.batch_decode(__snake_case    )
											self.assertListEqual(__snake_case       ,   __snake_case    )






							def UpperCAmelCase_      (       self       ,   __snake_case=6    ):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																			_SCREAMING_SNAKE_CASE					:			Dict							= self.rust_tokenizer_class.from_pretrained(__snake_case       ,   **__snake_case    )
																			# tokenizer_r.pad_token = None # Hotfixing padding = None
																			# Simple input
																			_SCREAMING_SNAKE_CASE					:			Tuple							= """This is a simple input"""
																			_SCREAMING_SNAKE_CASE					:			Optional[int]							= ["""This is a simple input 1""", """This is a simple input 2"""]
																			_SCREAMING_SNAKE_CASE					:			str							= ("""This is a simple input""", """This is a pair""")
																			_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [
																			    ("""This is a simple input 1""", """This is a simple input 2"""),
																			    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																			]

																			# Simple input tests
																			try:
																							tokenizer_r.encode(__snake_case       ,   max_length=__snake_case    )
																							tokenizer_r.encode_plus(__snake_case       ,   max_length=__snake_case    )

																							tokenizer_r.batch_encode_plus(__snake_case       ,   max_length=__snake_case    )
																							tokenizer_r.encode(__snake_case       ,   max_length=__snake_case    )
																							tokenizer_r.batch_encode_plus(__snake_case       ,   max_length=__snake_case    )
																			except ValueError:
																							self.fail("""Bloom Tokenizer should be able to deal with padding"""    )

																			_SCREAMING_SNAKE_CASE					:			str							= None  # Hotfixing padding = None
																			self.assertRaises(__snake_case       ,   tokenizer_r.encode       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""    )

																			# Simple input
																			self.assertRaises(__snake_case       ,   tokenizer_r.encode_plus       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""    )

																			# Simple input
																			self.assertRaises(
																			    __snake_case       ,   tokenizer_r.batch_encode_plus       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""       ,   )

																			# Pair input
																			self.assertRaises(__snake_case       ,   tokenizer_r.encode       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""    )

																			# Pair input
																			self.assertRaises(__snake_case       ,   tokenizer_r.encode_plus       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""    )

																			# Pair input
																			self.assertRaises(
																			    __snake_case       ,   tokenizer_r.batch_encode_plus       ,   __snake_case       ,   max_length=__snake_case       ,   padding="""max_length"""       ,   )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			str							= self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE					:			List[str]							= load_dataset("""xnli"""       ,   """all_languages"""       ,   split="""test"""       ,   streaming=__snake_case    )

											_SCREAMING_SNAKE_CASE					:			str							= next(iter(__snake_case    )    )["""premise"""]  # pick up one data
											_SCREAMING_SNAKE_CASE					:			Dict							= list(sample_data.values()    )

											_SCREAMING_SNAKE_CASE					:			str							= list(map(tokenizer.encode       ,   __snake_case    )    )
											_SCREAMING_SNAKE_CASE					:			Any							= [tokenizer.decode(__snake_case       ,   clean_up_tokenization_spaces=__snake_case    ) for x in output_tokens]
											self.assertListEqual(__snake_case       ,   __snake_case    )






							def UpperCAmelCase_      (       self    ):
											# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
											# any sequence length constraints. This test of the parent class will fail since it relies on the
											# maximum sequence length of the positoonal embeddings.
											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map    )       ,   1    )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()    )[0]    )       ,   1    )








'''simple docstring'''


def      snake_case_  (    SCREAMING_SNAKE_CASE__	):







				"""simple docstring"""





				return "".join(chr(ord(SCREAMING_SNAKE_CASE__	) - 32	) if """a""" <= char <= """z""" else char for char in word	)


if __name__ == "__main__":
				from doctest import testmod

				testmod()




