


"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class snake_case					(     _UpperCamelCase   , unittest.TestCase):
		__UpperCamelCase							=	ElectraTokenizer
		__UpperCamelCase							=	ElectraTokenizerFast
		__UpperCamelCase							=	True
		__UpperCamelCase							=	True
		__UpperCamelCase							=	filter_non_english







		def        a_       (    self					:				Tuple						)				->  str:





						'''simple docstring'''







						super().setUp()

						_A         =	[
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "[PAD]",
						    "[MASK]",
						    "want",
						    "##want",
						    "##ed",
						    "wa",
						    "un",
						    "runn",
						    "##ing",
						    ",",
						    "low",
						    "lowest",
						]
						_A         =	os.path.join(self.tmpdirname			,      VOCAB_FILES_NAMES["vocab_file"]						)
						with open(self.vocab_file			,      "w"			,      encoding="utf-8"						) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







		def        a_       (    self					:				str			,      a__					:				Any						)				->  List[Any]:





						'''simple docstring'''







						_A         =	"UNwant\u00E9d,running"
						_A         =	"unwanted, running"
						return input_text, output_text







		def        a_       (    self					:				List[Any]						)				->  str:





						'''simple docstring'''







						_A         =	self.tokenizer_class(self.vocab_file						)

						_A         =	tokenizer.tokenize("UNwant\u00E9d,running"						)
						self.assertListEqual(a__			,      ["un", "##want", "##ed", ",", "runn", "##ing"]						)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(a__						)			,      [9, 6, 7, 12, 10, 11]						)







		def        a_       (    self					:				Optional[int]						)				->  List[str]:





						'''simple docstring'''







						if not self.test_rust_tokenizer:
										return

						_A         =	self.get_tokenizer()
						_A         =	self.get_rust_tokenizer()

						_A         =	"UNwant\u00E9d,running"

						_A         =	tokenizer.tokenize(a__						)
						_A         =	rust_tokenizer.tokenize(a__						)
						self.assertListEqual(a__			,      a__						)

						_A         =	tokenizer.encode(a__			,      add_special_tokens=a__						)
						_A         =	rust_tokenizer.encode(a__			,      add_special_tokens=a__						)
						self.assertListEqual(a__			,      a__						)

						_A         =	self.get_rust_tokenizer()
						_A         =	tokenizer.encode(a__						)
						_A         =	rust_tokenizer.encode(a__						)
						self.assertListEqual(a__			,      a__						)

						# With lower casing
						_A         =	self.get_tokenizer(do_lower_case=a__						)
						_A         =	self.get_rust_tokenizer(do_lower_case=a__						)

						_A         =	"UNwant\u00E9d,running"

						_A         =	tokenizer.tokenize(a__						)
						_A         =	rust_tokenizer.tokenize(a__						)
						self.assertListEqual(a__			,      a__						)

						_A         =	tokenizer.encode(a__			,      add_special_tokens=a__						)
						_A         =	rust_tokenizer.encode(a__			,      add_special_tokens=a__						)
						self.assertListEqual(a__			,      a__						)

						_A         =	self.get_rust_tokenizer()
						_A         =	tokenizer.encode(a__						)
						_A         =	rust_tokenizer.encode(a__						)
						self.assertListEqual(a__			,      a__						)







		def        a_       (    self					:				Dict						)				->  Dict:





						'''simple docstring'''







						_A         =	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)			,      ["ah", "\u535A", "\u63A8", "zz"]						)







		def        a_       (    self					:				str						)				->  Union[str, Any]:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)			,      ["hello", "!", "how", "are", "you", "?"]						)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)			,      ["hello"]						)







		def        a_       (    self					:				List[str]						)				->  Dict:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__			,      strip_accents=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)			,      ["hällo", "!", "how", "are", "you", "?"]						)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)			,      ["h\u00E9llo"]						)







		def        a_       (    self					:				str						)				->  Any:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__			,      strip_accents=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)			,      ["hallo", "!", "how", "are", "you", "?"]						)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)			,      ["hello"]						)







		def        a_       (    self					:				Dict						)				->  Union[str, Any]:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)			,      ["hallo", "!", "how", "are", "you", "?"]						)
						self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)			,      ["hello"]						)







		def        a_       (    self					:				Optional[int]						)				->  List[Any]:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)			,      ["HeLLo", "!", "how", "Are", "yoU", "?"]						)







		def        a_       (    self					:				Dict						)				->  int:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__			,      strip_accents=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)			,      ["HäLLo", "!", "how", "Are", "yoU", "?"]						)







		def        a_       (    self					:				int						)				->  List[Any]:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__			,      strip_accents=a__						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)			,      ["HaLLo", "!", "how", "Are", "yoU", "?"]						)







		def        a_       (    self					:				int						)				->  Union[str, Any]:





						'''simple docstring'''







						_A         =	BasicTokenizer(do_lower_case=a__			,      never_split=["[UNK]"]						)

						self.assertListEqual(
						    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)			,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)







		def        a_       (    self					:				Union[str, Any]						)				->  Any:





						'''simple docstring'''







						_A         =	["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

						_A         =	{}
						for i, token in enumerate(a__						):
										_A         =	i
						_A         =	WordpieceTokenizer(vocab=a__			,      unk_token="[UNK]"						)

						self.assertListEqual(tokenizer.tokenize(""						)			,      []						)

						self.assertListEqual(tokenizer.tokenize("unwanted running"						)			,      ["un", "##want", "##ed", "runn", "##ing"]						)

						self.assertListEqual(tokenizer.tokenize("unwantedX running"						)			,      ["[UNK]", "runn", "##ing"]						)







		def        a_       (    self					:				Tuple						)				->  Optional[int]:





						'''simple docstring'''







						self.assertTrue(_is_whitespace(" "						)						)
						self.assertTrue(_is_whitespace("\t"						)						)
						self.assertTrue(_is_whitespace("\r"						)						)
						self.assertTrue(_is_whitespace("\n"						)						)
						self.assertTrue(_is_whitespace("\u00A0"						)						)

						self.assertFalse(_is_whitespace("A"						)						)
						self.assertFalse(_is_whitespace("-"						)						)







		def        a_       (    self					:				Any						)				->  Tuple:





						'''simple docstring'''







						self.assertTrue(_is_control("\u0005"						)						)

						self.assertFalse(_is_control("A"						)						)
						self.assertFalse(_is_control(" "						)						)
						self.assertFalse(_is_control("\t"						)						)
						self.assertFalse(_is_control("\r"						)						)







		def        a_       (    self					:				Union[str, Any]						)				->  Union[str, Any]:





						'''simple docstring'''







						self.assertTrue(_is_punctuation("-"						)						)
						self.assertTrue(_is_punctuation("$"						)						)
						self.assertTrue(_is_punctuation("`"						)						)
						self.assertTrue(_is_punctuation("."						)						)

						self.assertFalse(_is_punctuation("A"						)						)
						self.assertFalse(_is_punctuation(" "						)						)







		def        a_       (    self					:				List[Any]						)				->  Dict:





						'''simple docstring'''







						_A         =	self.get_tokenizer()
						_A         =	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(a__						) for t in ["Test", "\xad", "test"]]			,      [["[UNK]"], [], ["[UNK]"]]						)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(a__						) for t in ["Test", "\xad", "test"]]			,      [["[UNK]"], [], ["[UNK]"]]						)







		@slow
		def        a_       (    self					:				List[Any]						)				->  Optional[int]:





						'''simple docstring'''







						_A         =	self.tokenizer_class.from_pretrained("google/electra-base-discriminator"						)

						_A         =	tokenizer.encode("sequence builders"			,      add_special_tokens=a__						)
						_A         =	tokenizer.encode("multi-sequence build"			,      add_special_tokens=a__						)

						_A         =	tokenizer.build_inputs_with_special_tokens(a__						)
						_A         =	tokenizer.build_inputs_with_special_tokens(a__			,      a__						)

						assert encoded_sentence == [1_01] + text + [1_02]
						assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







		def        a_       (    self					:				Optional[int]						)				->  Optional[Any]:





						'''simple docstring'''







						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
														_A         =	self.rust_tokenizer_class.from_pretrained(a__			,      **a__						)

														_A         =	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														_A         =	tokenizer_r.encode_plus(
														    a__			,      return_attention_mask=a__			,      return_token_type_ids=a__			,      return_offsets_mapping=a__			,      add_special_tokens=a__			,      )

														_A         =	tokenizer_r.do_lower_case if hasattr(a__			,      "do_lower_case"						) else False
														_A         =	(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), "A"),
														        ((1, 2), ","),
														        ((3, 5), "na"),
														        ((5, 6), "##ï"),
														        ((6, 8), "##ve"),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), "Allen"),
														        ((21, 23), "##NL"),
														        ((23, 24), "##P"),
														        ((25, 33), "sentence"),
														        ((33, 34), "."),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), "a"),
														        ((1, 2), ","),
														        ((3, 8), "naive"),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), "allen"),
														        ((21, 23), "##nl"),
														        ((23, 24), "##p"),
														        ((25, 33), "sentence"),
														        ((33, 34), "."),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]			,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
														self.assertEqual([e[0] for e in expected_results]			,      tokens["offset_mapping"]						)







		def        a_       (    self					:				str						)				->  Optional[int]:





						'''simple docstring'''







						_A         =	["的", "人", "有"]
						_A         =	"".join(a__						)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
														_A         =	True
														_A         =	self.tokenizer_class.from_pretrained(a__			,      **a__						)
														_A         =	self.rust_tokenizer_class.from_pretrained(a__			,      **a__						)

														_A         =	tokenizer_p.encode(a__			,      add_special_tokens=a__						)
														_A         =	tokenizer_r.encode(a__			,      add_special_tokens=a__						)

														_A         =	tokenizer_r.convert_ids_to_tokens(a__						)
														_A         =	tokenizer_p.convert_ids_to_tokens(a__						)

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(a__			,      a__						)
														self.assertListEqual(a__			,      a__						)

														_A         =	False
														_A         =	self.rust_tokenizer_class.from_pretrained(a__			,      **a__						)
														_A         =	self.tokenizer_class.from_pretrained(a__			,      **a__						)

														_A         =	tokenizer_r.encode(a__			,      add_special_tokens=a__						)
														_A         =	tokenizer_p.encode(a__			,      add_special_tokens=a__						)

														_A         =	tokenizer_r.convert_ids_to_tokens(a__						)
														_A         =	tokenizer_p.convert_ids_to_tokens(a__						)

														# it is expected that only the first Chinese character is not preceded by "##".
														_A         =	[
														    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(a__						)
														]
														self.assertListEqual(a__			,      a__						)
														self.assertListEqual(a__			,      a__						)



"""simple docstring"""


from __future__ import annotations

a_											=			[2, 1, 4, 5, 6, 0, 8, 9, 1, 2, 0, 6, 4, 2, 0, 6, 5, 3, 2, 7]
class snake_case					:







		def __init__(    self					:				Optional[int]			,      a__					:				int						)				->  None:





						'''simple docstring'''







						_A         =	-1
						_A         =	-1
						_A         =	[-1] * length
						_A         =	None
						_A         =	None







		def __repr__(    self					:				Any						)				->  str:





						'''simple docstring'''







						return F"""Node(min_value={self.minn} max_value={self.maxx})"""






def    a__       ( __lowercase      )		->		Node | None:
				_A         =	Node(len(__lowercase      )      )
				_A     ,						_A         =	min(__lowercase      ), max(__lowercase      )
				# Leaf node case where the node contains only one unique value
				if root.minn == root.maxx:
								return root
				_A         =	(root.minn + root.maxx) // 2

				_A         =	[]
				_A         =	[]

				for index, num in enumerate(__lowercase      ):
								if num <= pivot:
												left_arr.append(__lowercase      )
								else:
												right_arr.append(__lowercase      )
								_A         =	len(__lowercase      )
				_A         =	build_tree(__lowercase      )
				_A         =	build_tree(__lowercase      )
				return root






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase      )		->		int:
				if index < 0 or node is None:
								return 0
				# Leaf node cases
				if node.minn == node.maxx:
								return index + 1 if node.minn == num else 0
				_A         =	(node.minn + node.maxx) // 2
				if num <= pivot:
								# go the left subtree and map index to the left subtree
								return rank_till_index(node.left   ,	__lowercase   ,	node.map_left[index] - 1      )
				else:
								# go to the right subtree and map index to the right subtree
								return rank_till_index(node.right   ,	__lowercase   ,	index - node.map_left[index]      )






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase      )		->		int:
				if start > end:
								return 0
				_A         =	rank_till_index(__lowercase   ,	__lowercase   ,	__lowercase      )
				_A         =	rank_till_index(__lowercase   ,	__lowercase   ,	start - 1      )
				return rank_till_end - rank_before_start






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase      )		->		int:
				if index > (end - start) or start > end or node is None:
								return -1
				# Leaf node case
				if node.minn == node.maxx:
								return node.minn
				# Number of elements in the left subtree in interval [start, end]
				_A         =	node.map_left[end] - (
				    node.map_left[start - 1] if start else 0
				)
				if num_elements_in_left_tree > index:
								return quantile(
								    node.left   ,	__lowercase   ,	(node.map_left[start - 1] if start else 0)   ,	node.map_left[end] - 1   ,	)
				else:
								return quantile(
								    node.right   ,	index - num_elements_in_left_tree   ,	start - (node.map_left[start - 1] if start else 0)   ,	end - node.map_left[end]   ,	)






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase      )		->		int:
				if (
				    start > end
				    or node is None
				    or start_num > end_num
				    or node.minn > end_num
				    or node.maxx < start_num
				):
								return 0
				if start_num <= node.minn and node.maxx <= end_num:
								return end - start + 1
				_A         =	range_counting(
				    node.left   ,	(node.map_left[start - 1] if start else 0)   ,	node.map_left[end] - 1   ,	__lowercase   ,	__lowercase   ,	)
				_A         =	range_counting(
				    node.right   ,	start - (node.map_left[start - 1] if start else 0)   ,	end - node.map_left[end]   ,	__lowercase   ,	__lowercase   ,	)
				return left + right


if __name__ == "__main__":
		import doctest

		doctest.testmod()