






"""simple docstring"""




import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_       :							Dict             =				get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')


@require_sentencepiece
@require_tokenizers
class 	__UpperCAmelCase		(				_lowerCAmelCase,  unittest.TestCase	):


	'''simple docstring'''





	lowercase		:	str							     =				SpeechTaTokenizer
	lowercase		:	Any							     =				False
	lowercase		:	Any							     =				True


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		super().setUp()

		# We have a SentencePiece fixture for testing
		_SCREAMING_SNAKE_CASE						=SpeechTaTokenizer(UpperCAmelCase_					)

		_SCREAMING_SNAKE_CASE						=AddedToken('''<mask>'''      ,      lstrip=UpperCAmelCase_      ,      rstrip=UpperCAmelCase_					)
		_SCREAMING_SNAKE_CASE						=mask_token
		tokenizer.add_special_tokens({'''mask_token''': mask_token}					)
		tokenizer.add_tokens(['''<ctc_blank>''']					)

		tokenizer.save_pretrained(self.tmpdirname					)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						='''this is a test'''
		_SCREAMING_SNAKE_CASE						='''this is a test'''
		return input_text, output_text


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A=False      ,      _A=2_0      ,      _A=5					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.get_input_output_texts(UpperCAmelCase_					)
		_SCREAMING_SNAKE_CASE						=tokenizer.encode(UpperCAmelCase_      ,      add_special_tokens=UpperCAmelCase_					)
		_SCREAMING_SNAKE_CASE						=tokenizer.decode(UpperCAmelCase_      ,      clean_up_tokenization_spaces=UpperCAmelCase_					)
		return text, ids


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						='''<pad>'''
		_SCREAMING_SNAKE_CASE						=1

		self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_					)      ,      UpperCAmelCase_					)
		self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_					)      ,      UpperCAmelCase_					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=list(self.get_tokenizer().get_vocab().keys()					)

		self.assertEqual(vocab_keys[0]      ,      '''<s>'''					)
		self.assertEqual(vocab_keys[1]      ,      '''<pad>'''					)
		self.assertEqual(vocab_keys[-4]      ,      '''œ'''					)
		self.assertEqual(vocab_keys[-2]      ,      '''<mask>'''					)
		self.assertEqual(vocab_keys[-1]      ,      '''<ctc_blank>'''					)
		self.assertEqual(len(UpperCAmelCase_					)      ,      8_1					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		self.assertEqual(self.get_tokenizer().vocab_size      ,      7_9					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.get_tokenizers(do_lower_case=UpperCAmelCase_					)
		for tokenizer in tokenizers:
			with self.subTest(f"""{tokenizer.__class__.__name__}"""					):
				_SCREAMING_SNAKE_CASE						=tokenizer.vocab_size
				_SCREAMING_SNAKE_CASE						=len(UpperCAmelCase_					)

				self.assertNotEqual(UpperCAmelCase_      ,      0					)

				# We usually have added tokens from the start in tests because our vocab fixtures are
				# smaller than the original vocabs - let's not assert this
				# self.assertEqual(vocab_size, all_size)

				_SCREAMING_SNAKE_CASE						=['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
				_SCREAMING_SNAKE_CASE						=tokenizer.add_tokens(UpperCAmelCase_					)
				_SCREAMING_SNAKE_CASE						=tokenizer.vocab_size
				_SCREAMING_SNAKE_CASE						=len(UpperCAmelCase_					)

				self.assertNotEqual(UpperCAmelCase_      ,      0					)
				self.assertEqual(UpperCAmelCase_      ,      UpperCAmelCase_					)
				self.assertEqual(UpperCAmelCase_      ,      len(UpperCAmelCase_					)					)
				self.assertEqual(UpperCAmelCase_      ,      all_size + len(UpperCAmelCase_					)					)

				_SCREAMING_SNAKE_CASE						=tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''      ,      add_special_tokens=UpperCAmelCase_					)

				self.assertGreaterEqual(len(UpperCAmelCase_					)      ,      4					)
				self.assertGreater(tokens[0]      ,      tokenizer.vocab_size - 1					)
				self.assertGreater(tokens[-3]      ,      tokenizer.vocab_size - 1					)

				_SCREAMING_SNAKE_CASE						={'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
				_SCREAMING_SNAKE_CASE						=tokenizer.add_special_tokens(UpperCAmelCase_					)
				_SCREAMING_SNAKE_CASE						=tokenizer.vocab_size
				_SCREAMING_SNAKE_CASE						=len(UpperCAmelCase_					)

				self.assertNotEqual(UpperCAmelCase_      ,      0					)
				self.assertEqual(UpperCAmelCase_      ,      UpperCAmelCase_					)
				self.assertEqual(UpperCAmelCase_      ,      len(UpperCAmelCase_					)					)
				self.assertEqual(UpperCAmelCase_      ,      all_size_a + len(UpperCAmelCase_					)					)

				_SCREAMING_SNAKE_CASE						=tokenizer.encode(
				    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''      ,      add_special_tokens=UpperCAmelCase_					)

				self.assertGreaterEqual(len(UpperCAmelCase_					)      ,      6					)
				self.assertGreater(tokens[0]      ,      tokenizer.vocab_size - 1					)
				self.assertGreater(tokens[0]      ,      tokens[1]					)
				self.assertGreater(tokens[-3]      ,      tokenizer.vocab_size - 1					)
				self.assertGreater(tokens[-3]      ,      tokens[-4]					)
				self.assertEqual(tokens[0]      ,      tokenizer.eos_token_id					)
				self.assertEqual(tokens[-3]      ,      tokenizer.pad_token_id					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.get_tokenizer()

		_SCREAMING_SNAKE_CASE						=tokenizer.tokenize('''This is a test'''					)
		# fmt: off
		self.assertListEqual(UpperCAmelCase_      ,      [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']					)
		# fmt: on

		self.assertListEqual(
		    tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)      ,      [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]      ,      )

		_SCREAMING_SNAKE_CASE						=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
		self.assertListEqual(
		    UpperCAmelCase_      ,      [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']					)

		_SCREAMING_SNAKE_CASE						=tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)
		# fmt: off
		self.assertListEqual(UpperCAmelCase_      ,      [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]					)
		# fmt: on

		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(UpperCAmelCase_					)
		self.assertListEqual(
		    UpperCAmelCase_      ,      [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']					)







	@slow
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=[
		    '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
		    '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
		    '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
		    '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
		    '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
		    '''conditioning on both left and right context in all layers.''',
		    '''The quick brown fox jumps over the lazy dog.''',
		]

		# fmt: off
		_SCREAMING_SNAKE_CASE						={
		    '''input_ids''': [
		        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
		        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
		        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
		    ],
		    '''attention_mask''': [
		        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
		        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
		        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
		    ]
		}
		# fmt: on

		self.tokenizer_integration_test_util(
		    expected_encoding=UpperCAmelCase_      ,      model_name='''microsoft/speecht5_asr'''      ,      revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''      ,      sequences=UpperCAmelCase_      ,      )






"""simple docstring"""


from collections import defaultdict
from typing import Optional

from ..image_utils import load_image
from ..utils import (
    add_end_docstrings,
    is_torch_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ChunkPipeline


if is_torch_available():
      import torch

      from ..models.auto.modeling_auto import MODEL_FOR_MASK_GENERATION_MAPPING

SCREAMING_SNAKE_CASE			  =  logging.get_logger(__name__)



@add_end_docstrings(_lowerCAmelCase						)
class 	__a	(   _lowerCAmelCase						):






       def __init__(	self			:				str							,   **UpperCAmelCase_			:				List[str]      )->	Tuple:







              """simple docstring"""
              super().__init__(**UpperCAmelCase_      )
              requires_backends(self							,   "vision"      )
              requires_backends(self							,   "torch"      )

              if self.framework != "pt":
                     raise ValueError(f"The {self.__class__} is only available in PyTorch."      )

              self.check_model_type(UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple							,   **UpperCAmelCase_			:				int      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		{}
              UpperCamelCase       			=		{}
              UpperCamelCase       			=		{}
              # preprocess args
              if "points_per_batch" in kwargs:
                     UpperCamelCase       			=		kwargs["points_per_batch"]
              if "points_per_crop" in kwargs:
                     UpperCamelCase       			=		kwargs["points_per_crop"]
              if "crops_n_layers" in kwargs:
                     UpperCamelCase       			=		kwargs["crops_n_layers"]
              if "crop_overlap_ratio" in kwargs:
                     UpperCamelCase       			=		kwargs["crop_overlap_ratio"]
              if "crop_n_points_downscale_factor" in kwargs:
                     UpperCamelCase       			=		kwargs["crop_n_points_downscale_factor"]
              # postprocess args
              if "pred_iou_thresh" in kwargs:
                     UpperCamelCase       			=		kwargs["pred_iou_thresh"]
              if "stability_score_offset" in kwargs:
                     UpperCamelCase       			=		kwargs["stability_score_offset"]
              if "mask_threshold" in kwargs:
                     UpperCamelCase       			=		kwargs["mask_threshold"]
              if "stability_score_thresh" in kwargs:
                     UpperCamelCase       			=		kwargs["stability_score_thresh"]
              if "crops_nms_thresh" in kwargs:
                     UpperCamelCase       			=		kwargs["crops_nms_thresh"]
              if "output_rle_mask" in kwargs:
                     UpperCamelCase       			=		kwargs["output_rle_mask"]
              if "output_bboxes_mask" in kwargs:
                     UpperCamelCase       			=		kwargs["output_bboxes_mask"]
              return preprocess_kwargs, forward_params, postprocess_kwargs






       def __call__(	self			:				Optional[int]							,   UpperCAmelCase_			:				Optional[int]							,   *UpperCAmelCase_			:				Tuple							,   UpperCAmelCase_			:				int=None							,   UpperCAmelCase_			:				Tuple=None							,   **UpperCAmelCase_			:				str      )->	Any:







              """simple docstring"""
              return super().__call__(UpperCAmelCase_							,   *UpperCAmelCase_							,   num_workers=UpperCAmelCase_							,   batch_size=UpperCAmelCase_							,   **UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]							,   UpperCAmelCase_			:				Optional[int]							,   UpperCAmelCase_			:				str=64							,   UpperCAmelCase_			:				int = 0							,   UpperCAmelCase_			:				float = 512 / 1_500							,   UpperCAmelCase_			:				Optional[int] = 32							,   UpperCAmelCase_			:				Optional[int] = 1							,   )->	List[Any]:







              """simple docstring"""
              UpperCamelCase       			=		load_image(UpperCAmelCase_      )
              UpperCamelCase       			=		self.image_processor.size["longest_edge"]
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.image_processor.generate_crop_boxes(
                  UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )
              UpperCamelCase       			=		self.image_processor(images=UpperCAmelCase_							,   return_tensors="pt"      )

              with self.device_placement():
                     if self.framework == "pt":
                            UpperCamelCase       			=		self.get_inference_context()
                            with inference_context():
                                   UpperCamelCase       			=		self._ensure_tensor_on_device(UpperCAmelCase_							,   device=self.device      )
                                   UpperCamelCase       			=		self.model.get_image_embeddings(model_inputs.pop("pixel_values"      )      )
                                   UpperCamelCase       			=		image_embeddings

              UpperCamelCase       			=		grid_points.shape[1]
              UpperCamelCase       			=		points_per_batch if points_per_batch is not None else n_points

              if points_per_batch <= 0:
                     raise ValueError(
                         "Cannot have points_per_batch<=0. Must be >=1 to returned batched outputs. "
                         "To return all points at once, set points_per_batch to None"      )

              for i in range(0							,   UpperCAmelCase_							,   UpperCAmelCase_      ):
                     UpperCamelCase       			=		grid_points[:, i : i + points_per_batch, :, :]
                     UpperCamelCase       			=		input_labels[:, i : i + points_per_batch]
                     UpperCamelCase       			=		i == n_points - points_per_batch
                     yield {
                         "input_points": batched_points,
                         "input_labels": labels,
                         "input_boxes": crop_boxes,
                         "is_last": is_last,
                         **model_inputs,
                     }






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]							,   UpperCAmelCase_			:				Union[str, Any]							,   UpperCAmelCase_			:				int=0.88							,   UpperCAmelCase_			:				Optional[int]=0.95							,   UpperCAmelCase_			:				Dict=0							,   UpperCAmelCase_			:				int=1							,   )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		model_inputs.pop("input_boxes"      )
              UpperCamelCase       			=		model_inputs.pop("is_last"      )
              UpperCamelCase       			=		model_inputs.pop("original_sizes"      ).tolist()
              UpperCamelCase       			=		model_inputs.pop("reshaped_input_sizes"      ).tolist()

              UpperCamelCase       			=		self.model(**UpperCAmelCase_      )

              # post processing happens here in order to avoid CPU GPU copies of ALL the masks
              UpperCamelCase       			=		model_outputs["pred_masks"]
              UpperCamelCase       			=		self.image_processor.post_process_masks(
                  UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   binarize=UpperCAmelCase_      )
              UpperCamelCase       			=		model_outputs["iou_scores"]
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.image_processor.filter_masks(
                  masks[0]							,   iou_scores[0]							,   original_sizes[0]							,   input_boxes[0]							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   )
              return {
                  "masks": masks,
                  "is_last": is_last,
                  "boxes": boxes,
                  "iou_scores": iou_scores,
              }






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any							,   UpperCAmelCase_			:				Tuple							,   UpperCAmelCase_			:				List[str]=False							,   UpperCAmelCase_			:				str=False							,   UpperCAmelCase_			:				Optional[int]=0.7							,   )->	Dict:







              """simple docstring"""
              UpperCamelCase       			=		[]
              UpperCamelCase       			=		[]
              UpperCamelCase       			=		[]
              for model_output in model_outputs:
                     all_scores.append(model_output.pop("iou_scores"      )      )
                     all_masks.extend(model_output.pop("masks"      )      )
                     all_boxes.append(model_output.pop("boxes"      )      )

              UpperCamelCase       			=		torch.cat(UpperCAmelCase_      )
              UpperCamelCase       			=		torch.cat(UpperCAmelCase_      )
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.image_processor.post_process_for_mask_generation(
                  UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )

              UpperCamelCase       			=		defaultdict(UpperCAmelCase_      )
              for output in model_outputs:
                     for k, v in output.items():
                            extra[k].append(UpperCAmelCase_      )

              UpperCamelCase       			=		{}
              if output_rle_mask:
                     UpperCamelCase       			=		rle_mask

              if output_bboxes_mask:
                     UpperCamelCase       			=		bounding_boxes

              return {"masks": output_masks, "scores": iou_scores, **optional, **extra}





