




"""simple docstring"""







import json
import os
import unittest
from typing import Tuple

from transformers import WavaVecaPhonemeCTCTokenizer
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.models.wavaveca_phoneme.tokenization_wavaveca_phoneme import WavaVecaPhonemeCTCTokenizerOutput
from transformers.testing_utils import require_phonemizer

from ...test_tokenization_common import TokenizerTesterMixin






@require_phonemizer
class       SCREAMING_SNAKE_CASE__					(					_a						,				unittest.TestCase					):
						_a    						=						WavaVecaPhonemeCTCTokenizer
						_a    						=						False


						def   __lowercase (							self				:			List[str]					):
										super().setUp()

										lowerCAmelCase  =				(
										    """<s> <pad> </s> <unk> n s t ə l a i k d m ɛ ɾ e ɪ p o ɐ z ð f j v b ɹ ʁ ʊ iː r w ʌ u ɡ æ aɪ ʃ h ɔ ɑː """
										    """ŋ ɚ eɪ β uː y ɑ̃ oʊ ᵻ eː θ aʊ ts oː ɔ̃ ɣ ɜ ɑ dʒ əl x ɜː ç ʒ tʃ ɔː ɑːɹ ɛ̃ ʎ ɔːɹ ʋ aː ɕ œ ø oːɹ ɲ yː """
										    """ʔ iə i5 s. tɕ ?? nʲ ɛː œ̃ ɭ ɔø ʑ tʲ ɨ ɛɹ ts. rʲ ɪɹ ɭʲ i.5 ɔɪ q sʲ u5 ʊɹ iɜ a5 iɛ5 øː ʕ ja əɜ th ɑ5 """
										    """oɪ dʲ ə5 tɕh ts.h mʲ ɯ dʑ vʲ e̞ tʃʲ ei5 o5 onɡ5 ɑu5 iɑ5 ai5 aɪɚ kh ə1 ʐ i2 ʉ ħ t[ aɪə ʲ ju ə2 u2 oɜ """
										    """pː iɛɜ ou5 y5 uɜ tː uo5 d[ uoɜ tsh ɑɜ ɵ i̪5 uei5 ɟ aɜ ɑɨ i.ɜ eʊ o2 ɐ̃ ä pʲ kʲ n̩ ɒ ph ɑu2 uɨ əɪ ɫ ɬ """
										    """yɜ bʲ ɑ2 s̪ aiɜ χ ɐ̃ʊ̃ 1 ə4 yæɜ a2 ɨː t̪ iouɜ ũ onɡɜ aɨ iɛ2 ɔɨ ɑuɜ o̞ ei2 iou2 c kː y2 ɖ oe dˤ yɛɜ """
										    """əʊ S ɡʲ onɡ2 u\" eiɜ ʈ ɯᵝ iou5 dZ r̝̊ i.2 tS s^ ʝ yə5 iɑɜ uə5 pf ɨu iɑ2 ou2 ər2 fʲ ai2 r̝ uəɜ ɳ əɨ """
										    """ua5 uɪ ɽ bː yu5 uo2 yɛ5 l̩ ɻ ərɜ ʂ i̪2 ouɜ uaɜ a. a.ː yæ5 dː r̩ ee ɪu ər5 i̪ ɜ æi u: i.ː t^ o1 ɪ^ """
										    """ai ueiɜ æː ɛɪ eə i. ɴ ie ua2 ɑ1 o4 tʃː o: ɑ: u1 N i̪1 au yæ2 u. qː yəɜ y: kʰ tʃʰ iʊ sx õ uo tʰ """
										    """uai5 bʰ u.ː uə2 ʊə d^ s̪ː yiɜ dʰ r. oe: i1 ɟː yu2 nʲʲ i̪4 uei2 tsʲ ɸ ĩ ɑ4 t̪ː eɑ u4 e: tsː ʈʰ ɡʰ """
										    """ɯɯ dʒʲ ʂʲ X ɵː uaiɜ tɕʲ ã t^ː ẽː yɛ2 cː i.1 ɛʊ dˤdˤ dʒː i4 ɡː yi ɕʲ ɟʰ pʰ dʑʲ yuɜ ua1 ua4 æiː ɐɐ """
										    """ui iou1 ʊː a1 iou4 cʰ iɛ1 yə2 ɖʰ ẽ ʒʲ ää ər4 iːː ɪː iɑ1 ər1 œː øi ɪuː cʰcʰ əː1 iː1 ũ kʰː o̞o̞ xʲ """
										    """ou1 iɛ4 e̞e̞ y1 dzː dʲʲ dʰː ɯᵝɯᵝ lː uo1 i.4 i: yɛ5ʲ a4"""
										).split(""" """					)
										lowerCAmelCase  =				dict(zip(lowerCAmelCase						,						range(len(lowerCAmelCase					)					)					)					)

										lowerCAmelCase  =				{"""pad_token""": """<pad>""", """unk_token""": """<unk>""", """bos_token""": """<s>""", """eos_token""": """</s>"""}

										lowerCAmelCase  =				os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES["""vocab_file"""]					)
										with open(self.vocab_file						,						"""w"""						,						encoding="""utf-8"""					) as fp:
														fp.write(json.dumps(lowerCAmelCase					) + """\n"""					)


						def   __lowercase (							self				:			Any						,						lowerCAmelCase				:			Optional[int]						,						lowerCAmelCase				:			List[str]=False						,						lowerCAmelCase				:			Optional[int]=20						,						lowerCAmelCase				:			Optional[int]=5					):
										lowerCAmelCase  =				[(i, tokenizer.decode([i]						,						clean_up_tokenization_spaces=lowerCAmelCase					)) for i in range(len(lowerCAmelCase					)					)]
										lowerCAmelCase  =				list(filter(lambda lowerCAmelCase					: [t[0]] == tokenizer.encode(t[1]						,						do_phonemize=lowerCAmelCase					)						,						lowerCAmelCase					)					)
										if max_length is not None and len(lowerCAmelCase					) > max_length:
														lowerCAmelCase  =				toks[:max_length]
										if min_length is not None and len(lowerCAmelCase					) < min_length and len(lowerCAmelCase					) > 0:
														while len(lowerCAmelCase					) < min_length:
																		lowerCAmelCase  =				toks + toks
        # toks_str = [t[1] for t in toks]
										lowerCAmelCase  =				[t[0] for t in toks]

										# Ensure consistency
										lowerCAmelCase  =				tokenizer.decode(lowerCAmelCase						,						clean_up_tokenization_spaces=lowerCAmelCase					)
										if " " not in output_txt and len(lowerCAmelCase					) > 1:
														lowerCAmelCase  =				(
														    tokenizer.decode([toks_ids[0]]						,						clean_up_tokenization_spaces=lowerCAmelCase					)
														    + """ """
														    + tokenizer.decode(toks_ids[1:]						,						clean_up_tokenization_spaces=lowerCAmelCase					)
														)
										if with_prefix_space:
														lowerCAmelCase  =				""" """ + output_txt
										lowerCAmelCase  =				tokenizer.encode(lowerCAmelCase						,						add_special_tokens=lowerCAmelCase					)
										return output_txt, output_ids


						def   __lowercase (							self				:			List[Any]						,						**lowerCAmelCase				:			List[Any]					):
										kwargs.update(self.special_tokens_map					)
										return WavaVecaPhonemeCTCTokenizer.from_pretrained(self.tmpdirname						,						**lowerCAmelCase					)


						def   __lowercase (							self				:			List[Any]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)

										# check adding a single token
										tokenizer.add_tokens("""xxx"""					)
										lowerCAmelCase  =				tokenizer("""m xxx ɪ"""						,						do_phonemize=lowerCAmelCase					).input_ids
										self.assertEqual(lowerCAmelCase						,						[13, 392, 17]					)  # xxx should be last token

										tokenizer.add_tokens(["""aaa""", """bbb""", """ccc"""]					)
										lowerCAmelCase  =				tokenizer("""m aaa ɪ ccc"""						,						do_phonemize=lowerCAmelCase					).input_ids
										self.assertEqual(lowerCAmelCase						,						[13, 393, 17, 395]					)  # aaa and ccc should be after xxx and 2 after aaa

										lowerCAmelCase  =				tokenizer("""maɪ c"""						,						do_phonemize=lowerCAmelCase					).input_ids
										self.assertEqual(lowerCAmelCase						,						[3, 200]					)  # mai should be <unk> (=3)


						def   __lowercase (							self				:			Optional[Any]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)
										self.assertEqual(lowerCAmelCase						,						"""h ə l oʊ h aʊ ɑːɹ j uː"""					)


						def   __lowercase (							self				:			Dict					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)
										self.assertEqual(tokenizer(lowerCAmelCase					).input_ids						,						tokenizer(lowerCAmelCase						,						do_phonemize=lowerCAmelCase					).input_ids					)


						def   __lowercase (							self				:			Dict					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)
										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)

										lowerCAmelCase  =				tokenizer.decode(tokenizer(lowerCAmelCase					).input_ids					)

										self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)


						def   __lowercase (							self				:			Optional[Any]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)

										lowerCAmelCase  =				[
										    [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98],
										    [24, 22, 5, 24, 22, 5, 77],
										]
										lowerCAmelCase  =				tokenizer.decode(sample_ids[0]					)
										lowerCAmelCase  =				tokenizer.batch_decode(lowerCAmelCase					)
										self.assertEqual(lowerCAmelCase						,						batch_tokens[0]					)
										self.assertEqual(lowerCAmelCase						,						["""k s ɾ ɾ l ɭʲ""", """j ð s j ð s oːɹ"""]					)


						def   __lowercase (							self				:			Union[str, Any]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)
										self.assertEqual(lowerCAmelCase						,						"""h ə l oʊ | h aʊ | ɑːɹ | j uː |"""					)


						def   __lowercase (							self				:			Tuple					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)
										self.assertEqual(tokenizer(lowerCAmelCase					).input_ids						,						tokenizer(lowerCAmelCase						,						do_phonemize=lowerCAmelCase					).input_ids					)


						def   __lowercase (							self				:			Optional[int]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										# fmt: off
										lowerCAmelCase  =				[
										    [11, 5, 15, tokenizer.pad_token_id, tokenizer.word_delimiter_token_id, 15, 8, tokenizer.word_delimiter_token_id, 98],
										    [tokenizer.word_delimiter_token_id, 24, 22, tokenizer.word_delimiter_token_id, 5, 24, 22, 5, 77],
										]
										# fmt: on

										# decode with word_del_token filter
										lowerCAmelCase  =				tokenizer.decode(sample_ids[0]					)
										lowerCAmelCase  =				tokenizer.batch_decode(lowerCAmelCase					)
										self.assertEqual(lowerCAmelCase						,						batch_tokens[0]					)
										self.assertEqual(lowerCAmelCase						,						["""k s ɾ ɾ l ɭʲ""", """j ð s j ð s oːɹ"""]					)

										# decode with no word_del_token filter
										lowerCAmelCase  =				tokenizer.decode(sample_ids[0]						,						filter_word_delimiter_token=lowerCAmelCase					)
										lowerCAmelCase  =				tokenizer.batch_decode(lowerCAmelCase						,						filter_word_delimiter_token=lowerCAmelCase					)
										self.assertEqual(lowerCAmelCase						,						batch_tokens[0]					)
										self.assertEqual(lowerCAmelCase						,						["""k s ɾ | ɾ l | ɭʲ""", """| j ð | s j ð s oːɹ"""]					)


						def   __lowercase (							self				:			Optional[int]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)

										lowerCAmelCase  =				tokenizer.decode(tokenizer(lowerCAmelCase					).input_ids						,						filter_word_delimiter_token=lowerCAmelCase					)

										self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)


						def   __lowercase (							self				:			List[Any]					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										lowerCAmelCase  =				"""Hello how are you"""
										lowerCAmelCase  =				tokenizer.phonemize(lowerCAmelCase						,						phonemizer_lang="""en-us"""					)

										lowerCAmelCase  =				tokenizer.decode(tokenizer(lowerCAmelCase					).input_ids						,						filter_word_delimiter_token=lowerCAmelCase					)

										self.assertEqual(""" """.join([p.strip() for p in phonemes.split(""" |"""					)]					).strip()						,						lowerCAmelCase					)


						def   __lowercase (							self				:			Dict					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained(
										    """facebook/wav2vec2-lv-60-espeak-cv-ft"""						,						word_delimiter_token=lowerCAmelCase					)
										lowerCAmelCase  =				"""Hello how are you"""

										lowerCAmelCase  =				tokenizer(lowerCAmelCase						,						phonemizer_lang="""en-us"""					).input_ids
										lowerCAmelCase  =				tokenizer(lowerCAmelCase						,						phonemizer_lang="""fr-fr"""					).input_ids

										self.assertNotEqual(lowerCAmelCase						,						lowerCAmelCase					)

										lowerCAmelCase  =				tokenizer.decode(lowerCAmelCase					)
										lowerCAmelCase  =				tokenizer.decode(lowerCAmelCase					)

										self.assertEqual(lowerCAmelCase						,						"""h ə l oʊ h aʊ ɑːɹ j uː"""					)
										self.assertEqual(lowerCAmelCase						,						"""ɛ l o h aʊ a ʁ j u"""					)


						def   __lowercase (							self				:			int					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)
										lowerCAmelCase  =				"""Hello how Are you"""
										lowerCAmelCase  =				"""hello how are you"""

										lowerCAmelCase  =				tokenizer(lowerCAmelCase					).input_ids
										lowerCAmelCase  =				tokenizer(lowerCAmelCase					).input_ids

										self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)


						def   __lowercase (							self				:			Any					):
										lowerCAmelCase  =				self.tokenizer_class.from_pretrained("""facebook/wav2vec2-lv-60-espeak-cv-ft"""					)
										tokenizer.add_tokens(["""!""", """?"""]					)
										tokenizer.add_special_tokens({"""cls_token""": """$$$"""}					)

										# fmt: off
										lowerCAmelCase  =				[
										    [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98, 392, 392, 393, 392, 392, 393, 394, 394],
										    [24, 22, 5, 24, 22, 5, 77, tokenizer.pad_token_id, 394, 394],
										]
										# fmt: on

										lowerCAmelCase  =				tokenizer.batch_decode(lowerCAmelCase					)
										self.assertEqual(lowerCAmelCase						,						["""k s ɾ ɾ l ɭʲ!?!? $$$""", """j ð s j ð s oːɹ $$$"""]					)


						@staticmethod
						def   __lowercase (							lowerCAmelCase				:			List[str]						,						lowerCAmelCase				:			Tuple					):
										lowerCAmelCase  =				[d[key] for d in offsets]
										return retrieved_list


						def   __lowercase (							self				:			Optional[Any]					):
										lowerCAmelCase  =				self.get_tokenizer(word_delimiter_token="""|"""					)
										tokenizer.add_tokens("""|"""					)

										# fmt: off
										# ksssɾɾ|ɾɾ<pad>ɾɾ|<pad>ɾlll|ɭʲ -> k s ɾ ɾ | ɾ l | ɭʲ"
										lowerCAmelCase  =				[11, 5, 5, 5, 15, 15, tokenizer.pad_token_id, 15, 15, tokenizer.word_delimiter_token_id, tokenizer.pad_token_id, 15, 8, 8, 8, tokenizer.word_delimiter_token_id, 98]
										# fmt: on

										lowerCAmelCase  =				tokenizer.decode(lowerCAmelCase						,						output_char_offsets=lowerCAmelCase						,						filter_word_delimiter_token=lowerCAmelCase					)
										# check Wav2Vec2CTCTokenizerOutput keys for char
										self.assertEqual(len(outputs.keys()					)						,						2					)
										self.assertTrue("""text""" in outputs					)
										self.assertTrue("""char_offsets""" in outputs					)
										self.assertTrue(isinstance(lowerCAmelCase						,						lowerCAmelCase					)					)

										# check that order of chars is correct and identical for both outputs
										self.assertEqual(""" """.join(self.get_from_offsets(outputs["""char_offsets"""]						,						"""char"""					)					)						,						outputs.text					)
										self.assertListEqual(
										    self.get_from_offsets(outputs["""char_offsets"""]						,						"""char"""					)						,						["""k""", """s""", """ɾ""", """ɾ""", """|""", """ɾ""", """l""", """|""", """ɭʲ"""]					)

										# check that offsets are actually correct for char
										# 0-1 is 11, 1-4 is 5, 4-6 is first 15, 6-7 is <pad> (thus not shown), 7-9 is second 15, 9-10 is word_delimiter_token,
										# 10-11 is <pad> (thus not shown), 11-12 is third 15, 12-15 is 8, 15-16 is word_delimiter_token, 16-17 is 98
										self.assertListEqual(
										    self.get_from_offsets(outputs["""char_offsets"""]						,						"""start_offset"""					)						,						[0, 1, 4, 7, 9, 11, 12, 15, 16]					)
										self.assertListEqual(
										    self.get_from_offsets(outputs["""char_offsets"""]						,						"""end_offset"""					)						,						[1, 4, 6, 9, 10, 12, 15, 16, 17]					)


						def   __lowercase (							self				:			Union[str, Any]					):
										lowerCAmelCase  =				self.get_tokenizer(word_delimiter_token="""|"""					)

										def check_list_tuples_equal(lowerCAmelCase				:			List[Any]						,						lowerCAmelCase				:			Tuple					):
														self.assertTrue(isinstance(lowerCAmelCase						,						lowerCAmelCase					)					)
														self.assertTrue(isinstance(outputs_list[0]						,						lowerCAmelCase					)					)

														# transform list to ModelOutput
														lowerCAmelCase  =				WavaVecaPhonemeCTCTokenizerOutput(
														    {k: [d[k] for d in outputs_list] for k in outputs_list[0]}					)

														self.assertListEqual(outputs_batch["""text"""]						,						outputs_batch_a["""text"""]					)

														def recursive_check(lowerCAmelCase				:			Tuple						,						lowerCAmelCase				:			Union[str, Any]					):
																		if isinstance(lowerCAmelCase						,						lowerCAmelCase					):
																						[recursive_check(lowerCAmelCase						,						lowerCAmelCase					) for la, la in zip(lowerCAmelCase						,						lowerCAmelCase					)]
																		self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)

														if "char_offsets" in outputs_batch:
																		recursive_check(outputs_batch["""char_offsets"""]						,						outputs_batch_a["""char_offsets"""]					)

        # fmt: off
										lowerCAmelCase  =				[
										    [11, 5, 15, tokenizer.pad_token_id, 15, 4, 8, 98, 32, 32, 32, 32, 4, 33, tokenizer.word_delimiter_token_id, 32, 32, 33, 34, 34],
										    [24, 22, 5, tokenizer.word_delimiter_token_id, tokenizer.word_delimiter_token_id, 24, 22, 22, 22, 4, 5, 77, tokenizer.pad_token_id, 22, 22, 4, 34, 34, 34, 34],
										]
										# fmt: on

										# We assume that `decode` works as expected. All we will check now is
										# the output type is correct and the output is identical to `decode`

										# char
										lowerCAmelCase  =				tokenizer.batch_decode(lowerCAmelCase						,						output_char_offsets=lowerCAmelCase					)
										lowerCAmelCase  =				[tokenizer.decode(lowerCAmelCase						,						output_char_offsets=lowerCAmelCase					) for ids in sample_ids]
										check_list_tuples_equal(lowerCAmelCase						,						lowerCAmelCase					)


						@unittest.skip("""Wav2Vec2PhonemeTokenizer always lower cases letters to correctly map to phonemes"""					)
						def   __lowercase (							self				:			Union[str, Any]					):
										pass


						@unittest.skip("""Wav2Vec2PhonemeTokenizer always puts spaces between phonemes"""					)
						def   __lowercase (							self				:			Dict					):
										pass


						@unittest.skip("""encodes to text to ids, but decodes ids to phonemes -> not possible to have internal consistency"""					)
						def   __lowercase (							self				:			int					):
										pass


						@unittest.skip("""Wav2Vec2PhonemeModel has no max model length => no testing"""					)
						def   __lowercase (							self				:			Union[str, Any]					):
										pass


						def   __lowercase (							self				:			Any					):
										lowerCAmelCase  =				self.get_tokenizers(do_lower_case=lowerCAmelCase					)
										for tokenizer in tokenizers:
														with self.subTest(f'''{tokenizer.__class__.__name__}'''					):
																		lowerCAmelCase  =				tokenizer.vocab_size
																		lowerCAmelCase  =				len(lowerCAmelCase					)

																		self.assertNotEqual(lowerCAmelCase						,						0					)

																		# We usually have added tokens from the start in tests because our vocab fixtures are
																		# smaller than the original vocabs - let's not assert this
																		# self.assertEqual(vocab_size, all_size)

																		lowerCAmelCase  =				["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
																		lowerCAmelCase  =				tokenizer.add_tokens(lowerCAmelCase					)
																		lowerCAmelCase  =				tokenizer.vocab_size
																		lowerCAmelCase  =				len(lowerCAmelCase					)

																		self.assertNotEqual(lowerCAmelCase						,						0					)
																		self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)
																		self.assertEqual(lowerCAmelCase						,						len(lowerCAmelCase					)					)
																		self.assertEqual(lowerCAmelCase						,						all_size + len(lowerCAmelCase					)					)

																		lowerCAmelCase  =				tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""						,						add_special_tokens=lowerCAmelCase					)

																		self.assertGreaterEqual(len(lowerCAmelCase					)						,						4					)
																		self.assertGreater(tokens[0]						,						tokenizer.vocab_size - 1					)
																		self.assertGreater(tokens[-3]						,						tokenizer.vocab_size - 1					)

																		lowerCAmelCase  =				{"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
																		lowerCAmelCase  =				tokenizer.add_special_tokens(lowerCAmelCase					)
																		lowerCAmelCase  =				tokenizer.vocab_size
																		lowerCAmelCase  =				len(lowerCAmelCase					)

																		self.assertNotEqual(lowerCAmelCase						,						0					)
																		self.assertEqual(lowerCAmelCase						,						lowerCAmelCase					)
																		self.assertEqual(lowerCAmelCase						,						len(lowerCAmelCase					)					)
																		self.assertEqual(lowerCAmelCase						,						all_size_a + len(lowerCAmelCase					)					)

																		lowerCAmelCase  =				tokenizer.encode(
																		    """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""						,						add_special_tokens=lowerCAmelCase					)

																		self.assertGreaterEqual(len(lowerCAmelCase					)						,						6					)
																		self.assertGreater(tokens[0]						,						tokenizer.vocab_size - 1					)
																		self.assertGreater(tokens[0]						,						tokens[1]					)
																		self.assertGreater(tokens[-3]						,						tokenizer.vocab_size - 1					)
																		self.assertGreater(tokens[-3]						,						tokens[-4]					)
																		self.assertEqual(tokens[0]						,						tokenizer.eos_token_id					)
																		self.assertEqual(tokens[-3]						,						tokenizer.pad_token_id					)


						@unittest.skip("""The tokenizer shouldn't be used to encode input IDs (except for labels), only to decode."""					)
						def   __lowercase (							self				:			Optional[Any]					):
										pass


						@unittest.skip("""The tokenizer shouldn't be used to encode input IDs (except for labels), only to decode."""					)
						def   __lowercase (							self				:			int					):
										pass




						def   __lowercase (							self				:			List[Any]					):
										# The default common tokenizer tests assumes that the output of `convert_tokens_to_string` is a string which
										# is not the case for Wav2Vec2PhonemeCTCTokenizer.
										lowerCAmelCase  =				self.get_tokenizers(fast=lowerCAmelCase						,						do_lower_case=lowerCAmelCase					)
										for tokenizer in tokenizers:
														with self.subTest(f'''{tokenizer.__class__.__name__}'''					):
																		lowerCAmelCase  =				["""ð""", """ɪ""", """s""", """ɪ""", """z""", """ɐ""", """t""", """ɛ""", """k""", """s""", """t"""]
																		lowerCAmelCase  =				tokenizer.convert_tokens_to_string(lowerCAmelCase					)

																		self.assertIsInstance(output["""text"""]						,						lowerCAmelCase					)











"""simple docstring"""







from collections import deque
from math import floor
from random import random
from time import time






class       SCREAMING_SNAKE_CASE__					:


						def __init__(							self				:			Union[str, Any]					):
										lowerCAmelCase  =				{}


						def   __lowercase (							self				:			Optional[int]						,						lowerCAmelCase				:			Union[str, Any]						,						lowerCAmelCase				:			Optional[Any]						,						lowerCAmelCase				:			Union[str, Any]=1					):
										if self.graph.get(lowerCAmelCase					):
														if self.graph[u].count([w, v]					) == 0:
																		self.graph[u].append([w, v]					)
										else:
														lowerCAmelCase  =				[[w, v]]
										if not self.graph.get(lowerCAmelCase					):
														lowerCAmelCase  =				[]


						def   __lowercase (							self				:			Optional[int]					):
										return list(self.graph					)


						def   __lowercase (							self				:			Optional[Any]						,						lowerCAmelCase				:			Optional[Any]						,						lowerCAmelCase				:			Dict					):
										if self.graph.get(lowerCAmelCase					):
														for _ in self.graph[u]:
																		if _[1] == v:
																						self.graph[u].remove(lowerCAmelCase					)


						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			Tuple=-2						,						lowerCAmelCase				:			List[Any]=-1					):
										if s == d:
														return []
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										if s == -2:
														lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				s

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if visited.count(node[1]					) < 1:
																										if node[1] == d:
																														visited.append(lowerCAmelCase					)
																														return visited
																										else:
																														stack.append(node[1]					)
																														visited.append(node[1]					)
																														lowerCAmelCase  =				node[1]
																														break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return visited


						def   __lowercase (							self				:			Tuple						,						lowerCAmelCase				:			Any=-1					):
										if c == -1:
														lowerCAmelCase  =				floor(random() * 1_0000					) + 10
										for i in range(lowerCAmelCase					):
														# every vertex has max 100 edges
														for _ in range(floor(random() * 102					) + 1					):
																		lowerCAmelCase  =				floor(random() * c					) + 1
																		if n != i:
																						self.add_pair(lowerCAmelCase						,						lowerCAmelCase						,						1					)


						def   __lowercase (							self				:			Optional[int]						,						lowerCAmelCase				:			List[str]=-2					):
										lowerCAmelCase  =				deque()
										lowerCAmelCase  =				[]
										if s == -2:
														lowerCAmelCase  =				list(self.graph					)[0]
										d.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										while d:
														lowerCAmelCase  =				d.popleft()
														if len(self.graph[s]					) != 0:
																		for node in self.graph[s]:
																						if visited.count(node[1]					) < 1:
																										d.append(node[1]					)
																										visited.append(node[1]					)
										return visited


						def   __lowercase (							self				:			List[Any]						,						lowerCAmelCase				:			Union[str, Any]					):
										lowerCAmelCase  =				0
										for x in self.graph:
														for y in self.graph[x]:
																		if y[1] == u:
																						count += 1
										return count


						def   __lowercase (							self				:			int						,						lowerCAmelCase				:			Tuple					):
										return len(self.graph[u]					)


						def   __lowercase (							self				:			Optional[int]						,						lowerCAmelCase				:			Optional[Any]=-2					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										if s == -2:
														lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				s
										lowerCAmelCase  =				[]

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if visited.count(node[1]					) < 1:
																										stack.append(node[1]					)
																										visited.append(node[1]					)
																										lowerCAmelCase  =				node[1]
																										break

            # check if all the children are visited
														if s == ss:
																		sorted_nodes.append(stack.pop()					)
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return sorted_nodes


						def   __lowercase (							self				:			Any					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				-2
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				s
										lowerCAmelCase  =				False
										lowerCAmelCase  =				set()

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if (
																						    visited.count(node[1]					) > 0
																						    and node[1] != parent
																						    and indirect_parents.count(node[1]					) > 0
																						    and not on_the_way_back
																						):
																										lowerCAmelCase  =				len(lowerCAmelCase					) - 1
																										while len_stack >= 0:
																														if stack[len_stack] == node[1]:
																																		anticipating_nodes.add(node[1]					)
																																		break
																														else:
																																		anticipating_nodes.add(stack[len_stack]					)
																																		len_stack -= 1
																						if visited.count(node[1]					) < 1:
																										stack.append(node[1]					)
																										visited.append(node[1]					)
																										lowerCAmelCase  =				node[1]
																										break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		lowerCAmelCase  =				True
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				False
																		indirect_parents.append(lowerCAmelCase					)
																		lowerCAmelCase  =				s
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return list(lowerCAmelCase					)


						def   __lowercase (							self				:			Tuple					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				-2
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				s
										lowerCAmelCase  =				False
										lowerCAmelCase  =				set()

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if (
																						    visited.count(node[1]					) > 0
																						    and node[1] != parent
																						    and indirect_parents.count(node[1]					) > 0
																						    and not on_the_way_back
																						):
																										lowerCAmelCase  =				len(lowerCAmelCase					) - 1
																										while len_stack_minus_one >= 0:
																														if stack[len_stack_minus_one] == node[1]:
																																		anticipating_nodes.add(node[1]					)
																																		break
																														else:
																																		return True
																						if visited.count(node[1]					) < 1:
																										stack.append(node[1]					)
																										visited.append(node[1]					)
																										lowerCAmelCase  =				node[1]
																										break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		lowerCAmelCase  =				True
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				False
																		indirect_parents.append(lowerCAmelCase					)
																		lowerCAmelCase  =				s
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return False


						def   __lowercase (							self				:			List[Any]						,						lowerCAmelCase				:			Any=-2						,						lowerCAmelCase				:			Tuple=-1					):
										lowerCAmelCase  =				time()
										self.dfs(lowerCAmelCase						,						lowerCAmelCase					)
										lowerCAmelCase  =				time()
										return end - begin




						def   __lowercase (							self				:			int						,						lowerCAmelCase				:			str=-2					):
										lowerCAmelCase  =				time()
										self.bfs(lowerCAmelCase					)
										lowerCAmelCase  =				time()
										return end - begin






class       SCREAMING_SNAKE_CASE__					:


						def __init__(							self				:			int					):
										lowerCAmelCase  =				{}


						def   __lowercase (							self				:			Optional[Any]						,						lowerCAmelCase				:			Tuple						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			str=1					):
										# check if the u exists
										if self.graph.get(lowerCAmelCase					):
														# if there already is a edge
														if self.graph[u].count([w, v]					) == 0:
																		self.graph[u].append([w, v]					)
										else:
														# if u does not exist
														lowerCAmelCase  =				[[w, v]]
										# add the other way
										if self.graph.get(lowerCAmelCase					):
														# if there already is a edge
														if self.graph[v].count([w, u]					) == 0:
																		self.graph[v].append([w, u]					)
										else:
														# if u does not exist
														lowerCAmelCase  =				[[w, u]]


						def   __lowercase (							self				:			Dict						,						lowerCAmelCase				:			Any						,						lowerCAmelCase				:			int					):
										if self.graph.get(lowerCAmelCase					):
														for _ in self.graph[u]:
																		if _[1] == v:
																						self.graph[u].remove(lowerCAmelCase					)
        # the other way round
										if self.graph.get(lowerCAmelCase					):
														for _ in self.graph[v]:
																		if _[1] == u:
																						self.graph[v].remove(lowerCAmelCase					)


						def   __lowercase (							self				:			Dict						,						lowerCAmelCase				:			Optional[int]=-2						,						lowerCAmelCase				:			Optional[Any]=-1					):
										if s == d:
														return []
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										if s == -2:
														lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				s

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if visited.count(node[1]					) < 1:
																										if node[1] == d:
																														visited.append(lowerCAmelCase					)
																														return visited
																										else:
																														stack.append(node[1]					)
																														visited.append(node[1]					)
																														lowerCAmelCase  =				node[1]
																														break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return visited


						def   __lowercase (							self				:			Any						,						lowerCAmelCase				:			Dict=-1					):
										if c == -1:
														lowerCAmelCase  =				floor(random() * 1_0000					) + 10
										for i in range(lowerCAmelCase					):
														# every vertex has max 100 edges
														for _ in range(floor(random() * 102					) + 1					):
																		lowerCAmelCase  =				floor(random() * c					) + 1
																		if n != i:
																						self.add_pair(lowerCAmelCase						,						lowerCAmelCase						,						1					)


						def   __lowercase (							self				:			int						,						lowerCAmelCase				:			Tuple=-2					):
										lowerCAmelCase  =				deque()
										lowerCAmelCase  =				[]
										if s == -2:
														lowerCAmelCase  =				list(self.graph					)[0]
										d.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										while d:
														lowerCAmelCase  =				d.popleft()
														if len(self.graph[s]					) != 0:
																		for node in self.graph[s]:
																						if visited.count(node[1]					) < 1:
																										d.append(node[1]					)
																										visited.append(node[1]					)
										return visited


						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			str					):
										return len(self.graph[u]					)


						def   __lowercase (							self				:			Optional[Any]					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				-2
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				s
										lowerCAmelCase  =				False
										lowerCAmelCase  =				set()

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if (
																						    visited.count(node[1]					) > 0
																						    and node[1] != parent
																						    and indirect_parents.count(node[1]					) > 0
																						    and not on_the_way_back
																						):
																										lowerCAmelCase  =				len(lowerCAmelCase					) - 1
																										while len_stack >= 0:
																														if stack[len_stack] == node[1]:
																																		anticipating_nodes.add(node[1]					)
																																		break
																														else:
																																		anticipating_nodes.add(stack[len_stack]					)
																																		len_stack -= 1
																						if visited.count(node[1]					) < 1:
																										stack.append(node[1]					)
																										visited.append(node[1]					)
																										lowerCAmelCase  =				node[1]
																										break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		lowerCAmelCase  =				True
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				False
																		indirect_parents.append(lowerCAmelCase					)
																		lowerCAmelCase  =				s
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return list(lowerCAmelCase					)


						def   __lowercase (							self				:			int					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				list(self.graph					)[0]
										stack.append(lowerCAmelCase					)
										visited.append(lowerCAmelCase					)
										lowerCAmelCase  =				-2
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				s
										lowerCAmelCase  =				False
										lowerCAmelCase  =				set()

										while True:
														# check if there is any non isolated nodes
														if len(self.graph[s]					) != 0:
																		lowerCAmelCase  =				s
																		for node in self.graph[s]:
																						if (
																						    visited.count(node[1]					) > 0
																						    and node[1] != parent
																						    and indirect_parents.count(node[1]					) > 0
																						    and not on_the_way_back
																						):
																										lowerCAmelCase  =				len(lowerCAmelCase					) - 1
																										while len_stack_minus_one >= 0:
																														if stack[len_stack_minus_one] == node[1]:
																																		anticipating_nodes.add(node[1]					)
																																		break
																														else:
																																		return True
																						if visited.count(node[1]					) < 1:
																										stack.append(node[1]					)
																										visited.append(node[1]					)
																										lowerCAmelCase  =				node[1]
																										break

            # check if all the children are visited
														if s == ss:
																		stack.pop()
																		lowerCAmelCase  =				True
																		if len(lowerCAmelCase					) != 0:
																						lowerCAmelCase  =				stack[len(lowerCAmelCase					) - 1]
														else:
																		lowerCAmelCase  =				False
																		indirect_parents.append(lowerCAmelCase					)
																		lowerCAmelCase  =				s
																		lowerCAmelCase  =				ss

														# check if se have reached the starting point
														if len(lowerCAmelCase					) == 0:
																		return False


						def   __lowercase (							self				:			str					):
										return list(self.graph					)


						def   __lowercase (							self				:			Dict						,						lowerCAmelCase				:			Dict=-2						,						lowerCAmelCase				:			List[str]=-1					):
										lowerCAmelCase  =				time()
										self.dfs(lowerCAmelCase						,						lowerCAmelCase					)
										lowerCAmelCase  =				time()
										return end - begin




						def   __lowercase (							self				:			str						,						lowerCAmelCase				:			Union[str, Any]=-2					):
										lowerCAmelCase  =				time()
										self.bfs(lowerCAmelCase					)
										lowerCAmelCase  =				time()
										return end - begin





