





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class        __SCREAMING_SNAKE_CASE					(     _a   ,		unittest.TestCase       ):
							snake_case     :      Union[str, Any]						     =       None
							snake_case     :      int						     =       BloomTokenizerFast
							snake_case     :      Union[str, Any]						     =       BloomTokenizerFast
							snake_case     :      List[Any]						     =       True
							snake_case     :      List[str]						     =       False
							snake_case     :      Dict						     =       """tokenizer_file"""
							snake_case     :      Optional[Any]						     =       {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






							def 				_lowerCamelCase		(						self  ):
														super().setUp()
														UpperCamelCase__				     =			BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""  )
														tokenizer.save_pretrained(self.tmpdirname  )






							def 				_lowerCamelCase		(						self  , **__lowerCAmelCase  ):
														kwargs.update(self.special_tokens_map  )
														return BloomTokenizerFast.from_pretrained(self.tmpdirname  , **__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.get_rust_tokenizer()

														UpperCamelCase__				     =			["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
														UpperCamelCase__				     =			[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

														UpperCamelCase__				     =			tokenizer.batch_encode_plus(__lowerCAmelCase  )["""input_ids"""]
														self.assertListEqual(__lowerCAmelCase  , __lowerCAmelCase  )

														UpperCamelCase__				     =			tokenizer.batch_decode(__lowerCAmelCase  )
														self.assertListEqual(__lowerCAmelCase  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase=6  ):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																												UpperCamelCase__				     =			self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase  , **__lowerCAmelCase  )
																												# tokenizer_r.pad_token = None # Hotfixing padding = None
																												# Simple input
																												UpperCamelCase__				     =			"""This is a simple input"""
																												UpperCamelCase__				     =			["""This is a simple input 1""", """This is a simple input 2"""]
																												UpperCamelCase__				     =			("""This is a simple input""", """This is a pair""")
																												UpperCamelCase__				     =			[
																												    ("""This is a simple input 1""", """This is a simple input 2"""),
																												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																												]

																												# Simple input tests
																												try:
																																			tokenizer_r.encode(__lowerCAmelCase  , max_length=__lowerCAmelCase  )
																																			tokenizer_r.encode_plus(__lowerCAmelCase  , max_length=__lowerCAmelCase  )

																																			tokenizer_r.batch_encode_plus(__lowerCAmelCase  , max_length=__lowerCAmelCase  )
																																			tokenizer_r.encode(__lowerCAmelCase  , max_length=__lowerCAmelCase  )
																																			tokenizer_r.batch_encode_plus(__lowerCAmelCase  , max_length=__lowerCAmelCase  )
																												except ValueError:
																																			self.fail("""Bloom Tokenizer should be able to deal with padding"""  )

																												UpperCamelCase__				     =			None  # Hotfixing padding = None
																												self.assertRaises(__lowerCAmelCase  , tokenizer_r.encode  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  )

																												# Simple input
																												self.assertRaises(__lowerCAmelCase  , tokenizer_r.encode_plus  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  )

																												# Simple input
																												self.assertRaises(
																												    __lowerCAmelCase  , tokenizer_r.batch_encode_plus  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  , )

																												# Pair input
																												self.assertRaises(__lowerCAmelCase  , tokenizer_r.encode  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  )

																												# Pair input
																												self.assertRaises(__lowerCAmelCase  , tokenizer_r.encode_plus  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  )

																												# Pair input
																												self.assertRaises(
																												    __lowerCAmelCase  , tokenizer_r.batch_encode_plus  , __lowerCAmelCase  , max_length=__lowerCAmelCase  , padding="""max_length"""  , )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.get_rust_tokenizer()
														UpperCamelCase__				     =			load_dataset("""xnli"""  , """all_languages"""  , split="""test"""  , streaming=__lowerCAmelCase  )

														UpperCamelCase__				     =			next(iter(__lowerCAmelCase  )  )["""premise"""]  # pick up one data
														UpperCamelCase__				     =			list(sample_data.values()  )

														UpperCamelCase__				     =			list(map(tokenizer.encode  , __lowerCAmelCase  )  )
														UpperCamelCase__				     =			[tokenizer.decode(__lowerCAmelCase  , clean_up_tokenization_spaces=__lowerCAmelCase  ) for x in output_tokens]
														self.assertListEqual(__lowerCAmelCase  , __lowerCAmelCase  )







							def 				_lowerCamelCase		(						self  ):
														# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
														# any sequence length constraints. This test of the parent class will fail since it relies on the
														# maximum sequence length of the positoonal embeddings.
														self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )  , 1  )
														self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )  , 1  )








from typing import Any





def     _UpperCamelCase     (a__   :list       ):






							"""simple docstring"""





							if not input_list:
														return []
							UpperCamelCase__				     =			[input_list.count(a__       ) for value in input_list]
							UpperCamelCase__				     =			max(a__       )  # Gets the maximum count in the input list.
							# Gets values of modes
							return sorted({input_list[i] for i, value in enumerate(a__       ) if value == y}       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




