


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase       		=    get_tests_dir("""fixtures/spiece.model""")




@require_sentencepiece
@require_tokenizers
class 							lowerCamelCase   (  _UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		Union[str, Any]										=						AlbertTokenizer
	_lowerCAmelCase     :		List[str]										=						AlbertTokenizerFast
	_lowerCAmelCase     :		Dict										=						True
	_lowerCAmelCase     :		Any										=						True
	_lowerCAmelCase     :		int										=						True


	def  A(    self):
							super().setUp()

							# We have a SentencePiece fixture for testing
							__UpperCAmelCase			:							Optional[Any]	    =	AlbertTokenizer(lowercase__)
							tokenizer.save_pretrained(self.tmpdirname)


	def  A(    self				,     lowercase__):
							__UpperCAmelCase			:							Tuple	    =	'this is a test'
							__UpperCAmelCase			:							List[str]	    =	'this is a test'
							return input_text, output_text


	def  A(    self):
							__UpperCAmelCase			:							Optional[int]	    =	'<pad>'
							__UpperCAmelCase			:							str	    =	0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__)				,     lowercase__)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__)				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							List[Any]	    =	list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,     '''<pad>''')
							self.assertEqual(vocab_keys[1]				,     '''<unk>''')
							self.assertEqual(vocab_keys[-1]				,     '''▁eloquent''')
							self.assertEqual(len(lowercase__)				,     3_0_0_0_0)


	def  A(    self):
							self.assertEqual(self.get_tokenizer().vocab_size				,     3_0_0_0_0)


	def  A(    self):
							if not self.test_rust_tokenizer:
													return

							__UpperCAmelCase			:							int	    =	self.get_tokenizer()
							__UpperCAmelCase			:							List[str]	    =	self.get_rust_tokenizer()

							__UpperCAmelCase			:							Optional[int]	    =	'I was born in 92000, and this is falsé.'

							__UpperCAmelCase			:							int	    =	tokenizer.tokenize(lowercase__)
							__UpperCAmelCase			:							Union[str, Any]	    =	rust_tokenizer.tokenize(lowercase__)
							self.assertListEqual(lowercase__				,     lowercase__)

							__UpperCAmelCase			:							List[str]	    =	tokenizer.encode(lowercase__				,     add_special_tokens=lowercase__)
							__UpperCAmelCase			:							int	    =	rust_tokenizer.encode(lowercase__				,     add_special_tokens=lowercase__)
							self.assertListEqual(lowercase__				,     lowercase__)

							__UpperCAmelCase			:							List[Any]	    =	self.get_rust_tokenizer()
							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.encode(lowercase__)
							__UpperCAmelCase			:							int	    =	rust_tokenizer.encode(lowercase__)
							self.assertListEqual(lowercase__				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							List[Any]	    =	AlbertTokenizer(lowercase__				,     keep_accents=lowercase__)

							__UpperCAmelCase			:							int	    =	tokenizer.tokenize('''This is a test''')
							self.assertListEqual(lowercase__				,     ['''▁this''', '''▁is''', '''▁a''', '''▁test'''])

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)				,     [4_8, 2_5, 2_1, 1_2_8_9])

							__UpperCAmelCase			:							Any	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.'''])
							__UpperCAmelCase			:							Any	    =	tokenizer.convert_tokens_to_ids(lowercase__)
							self.assertListEqual(lowercase__				,     [3_1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9])

							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.convert_ids_to_tokens(lowercase__)
							self.assertListEqual(
							    lowercase__				,     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']				,     )


	def  A(    self):
							__UpperCAmelCase			:							int	    =	AlbertTokenizer(lowercase__)

							__UpperCAmelCase			:							Any	    =	tokenizer.encode('''sequence builders''')
							__UpperCAmelCase			:							str	    =	tokenizer.encode('''multi-sequence build''')

							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__)
							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__				,     lowercase__)

							assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
							assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
							    tokenizer.sep_token_id
							]


	@slow
	def  A(    self):
							# fmt: off
							__UpperCAmelCase			:							Optional[int]	    =	{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1_9_7_0, 1_3, 5, 6_0_9_2, 1_6_7, 2_8, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 7_0_2_8, 1_2_0_5_1, 1_8, 1_7, 7_1_0_3, 2_1_5_3, 6_7_3, 8, 3_5_1_5, 1_8_6_8_4, 8, 4_4_6_1, 6, 1_9_2_7, 2_9_7, 8, 1_2_0_6_0, 2_6_0_7, 1_8, 1_3, 5, 4_4_6_1, 1_5, 1_0_5_3_8, 3_8, 8, 1_3_5, 1_5, 8_2_2, 5_8, 1_5, 9_9_3, 1_0_3_6_3, 1_5, 1_4_6_0, 8_0_0_5, 4_4_6_1, 1_5, 9_9_3, 2_5_5, 2_3_2_8, 9, 9, 9, 6, 2_6, 1_1_1_2, 8_1_6, 3_2_6_0, 1_3, 5, 1_0_3, 2_3_7_7, 6, 1_7, 1_1_1_2, 8_1_6, 2_7_8_2, 1_3, 5, 1_0_3, 1_0_6_4_1, 6, 2_9, 8_4, 2_5_1_2, 2_4_3_0, 7_8_2, 1_8_6_8_4, 2_7_6_1, 1_9, 8_0_8, 2_4_3_0, 2_5_5_6, 1_7, 8_5_5, 1_4_8_0, 9_4_7_7, 4_0_9_1, 1_2_8, 1_1_7_1_2, 1_5, 7_1_0_3, 2_1_5_3, 6_7_3, 1_7, 2_4_8_8_3, 9_9_9_0, 9, 3], [2, 1_1_5_0_2, 2_5, 1_0_0_6, 2_0, 7_8_2, 8, 1_1_8_0_9, 8_5_5, 1_7_3_2, 1_9_3_9_3, 1_8_6_6_7, 3_7, 3_6_7, 2_1_0_1_8, 6_9, 1_8_5_4, 3_4, 1_1_8_6_0, 1_9_1_2_4, 2_7, 1_5_6, 2_2_5, 1_7, 1_9_3, 4_1_4_1, 1_9, 6_5, 9_1_2_4, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 1_4, 2_2_3_1, 8_8_6, 2_3_8_5, 1_7_6_5_9, 8_4, 1_4, 1_6_7_9_2, 1_9_5_2, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowercase__				,     model_name='''albert-base-v2'''				,     revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''				,     )









"""simple docstring"""


import heapq as hq
import math
from collections.abc import Iterator




class 						__UpperCamelCase     :
					def __init__(							self							:	Union[str, Any]     , UpperCAmelCase							:	List[Any]   )	->       List[str]:
											lowerCAmelCase   :Dict						=			str(id_   )
											lowerCAmelCase   :str						=			None
											lowerCAmelCase   :Any						=			None
											lowerCAmelCase   :List[str]						=			[]
											lowerCAmelCase   :List[Any]						=			{}  # {vertex:distance}
					def __lt__(							self							:	Union[str, Any]     , UpperCAmelCase							:	Any   )	->       str:
											return self.key < other.key
					def __repr__(							self							:	Union[str, Any]   )	->       Dict:
											return self.id
					def        UpperCAmelCase__ (							self							:	Any     , UpperCAmelCase							:	Tuple   )	->       List[str]:
											self.neighbors.append(UpperCAmelCase   )
					def        UpperCAmelCase__ (							self							:	str     , UpperCAmelCase							:	Optional[Any]     , UpperCAmelCase							:	List[Any]   )	->       Optional[int]:
											lowerCAmelCase   :str						=			weight


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						graph[a - 1].add_neighbor(graph[b - 1] )
						graph[b - 1].add_neighbor(graph[a - 1] )
						# add the edges:
						graph[a - 1].add_edge(graph[b - 1]     ,      a__ )
						graph[b - 1].add_edge(graph[a - 1]     ,      a__ )


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :List[str]						=			[]
						for u in graph:
												lowerCAmelCase   :Tuple						=			math.inf
												lowerCAmelCase   :Any						=			None
						lowerCAmelCase   :Dict						=			0
						lowerCAmelCase   :int						=			graph[:]
						while q:
												lowerCAmelCase   :str						=			min(a__ )
												q.remove(a__ )
												for v in u.neighbors:
																		if (v in q) and (u.edges[v.id] < v.key):
																								lowerCAmelCase   :List[str]						=			u
																								lowerCAmelCase   :List[Any]						=			u.edges[v.id]
						for i in range(1     ,      len(a__ ) ):
												a.append((int(graph[i].id ) + 1, int(graph[i].pi.id ) + 1) )
						return a


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						for u in graph:
												lowerCAmelCase   :List[str]						=			math.inf
												lowerCAmelCase   :str						=			None
						lowerCAmelCase   :Optional[int]						=			0

						lowerCAmelCase   :Dict						=			list(a__ )
						hq.heapify(a__ )

						while h:
												lowerCAmelCase   :Union[str, Any]						=			hq.heappop(a__ )
												for v in u.neighbors:
																		if (v in h) and (u.edges[v.id] < v.key):
																								lowerCAmelCase   :int						=			u
																								lowerCAmelCase   :List[Any]						=			u.edges[v.id]
																								hq.heapify(a__ )

						for i in range(1     ,      len(a__ ) ):
												yield (int(graph[i].id ) + 1, int(graph[i].pi.id ) + 1)


def        UpperCAmelCase      (					):





						'''simple docstring'''








if __name__ == "__main__":
		import doctest

		doctest.testmod()