






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 						UpperCAmelCase_  (		lowerCamelCase_    ,     unittest.TestCase    ):


						'''simple docstring'''




						A   :      Optional[int]						 =    ElectraTokenizer
						A   :      Tuple						 =    ElectraTokenizerFast
						A   :      List[Any]						 =    True
						A   :      Optional[Any]						 =    True
						A   :      int						 =    filter_non_english






						def 					_lowerCAmelCase				(		self )       -> Optional[int]:
								super().setUp()

								snake_case_ :							int         =				[
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    '[PAD]',
								    '[MASK]',
								    'want',
								    '##want',
								    '##ed',
								    'wa',
								    'un',
								    'runn',
								    '##ing',
								    ',',
								    'low',
								    'lowest',
								]
								snake_case_ :							str         =				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["vocab_file"] )
								with open(self.vocab_file						, "w"						, encoding="utf-8" ) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )






						def 					_lowerCAmelCase				(		self						, _SCREAMING_SNAKE_CASE )       -> Optional[int]:
								snake_case_ :							Optional[int]         =				'UNwant\u00E9d,running'
								snake_case_ :							List[str]         =				'unwanted, running'
								return input_text, output_text






						def 					_lowerCAmelCase				(		self )       -> Optional[Any]:
								snake_case_ :							Optional[int]         =				self.tokenizer_class(self.vocab_file )

								snake_case_ :							int         =				tokenizer.tokenize("UNwant\u00E9d,running" )
								self.assertListEqual(snake_case_						, ["un", "##want", "##ed", ",", "runn", "##ing"] )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_ )						, [9, 6, 7, 12, 10, 11] )






						def 					_lowerCAmelCase				(		self )       -> Optional[Any]:

								if not self.test_rust_tokenizer:
										return

								snake_case_ :							Tuple         =				self.get_tokenizer()
								snake_case_ :							Union[str, Any]         =				self.get_rust_tokenizer()

								snake_case_ :							Tuple         =				'UNwant\u00E9d,running'

								snake_case_ :							Dict         =				tokenizer.tokenize(snake_case_ )
								snake_case_ :							Union[str, Any]         =				rust_tokenizer.tokenize(snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )

								snake_case_ :							Any         =				tokenizer.encode(snake_case_						, add_special_tokens=snake_case_ )
								snake_case_ :							Dict         =				rust_tokenizer.encode(snake_case_						, add_special_tokens=snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )

								snake_case_ :							int         =				self.get_rust_tokenizer()
								snake_case_ :							Dict         =				tokenizer.encode(snake_case_ )
								snake_case_ :							List[str]         =				rust_tokenizer.encode(snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )

								# With lower casing
								snake_case_ :							Any         =				self.get_tokenizer(do_lower_case=snake_case_ )
								snake_case_ :							Optional[int]         =				self.get_rust_tokenizer(do_lower_case=snake_case_ )

								snake_case_ :							Tuple         =				'UNwant\u00E9d,running'

								snake_case_ :							List[str]         =				tokenizer.tokenize(snake_case_ )
								snake_case_ :							str         =				rust_tokenizer.tokenize(snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )

								snake_case_ :							Dict         =				tokenizer.encode(snake_case_						, add_special_tokens=snake_case_ )
								snake_case_ :							Any         =				rust_tokenizer.encode(snake_case_						, add_special_tokens=snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )

								snake_case_ :							Optional[int]         =				self.get_rust_tokenizer()
								snake_case_ :							int         =				tokenizer.encode(snake_case_ )
								snake_case_ :							Optional[int]         =				rust_tokenizer.encode(snake_case_ )
								self.assertListEqual(snake_case_						, snake_case_ )






						def 					_lowerCAmelCase				(		self )       -> Dict:
								snake_case_ :							List[str]         =				BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz" )						, ["ah", "\u535A", "\u63A8", "zz"] )






						def 					_lowerCAmelCase				(		self )       -> Tuple:
								snake_case_ :							Optional[Any]         =				BasicTokenizer(do_lower_case=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )						, ["hello", "!", "how", "are", "you", "?"] )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )						, ["hello"] )






						def 					_lowerCAmelCase				(		self )       -> Optional[int]:
								snake_case_ :							Union[str, Any]         =				BasicTokenizer(do_lower_case=snake_case_						, strip_accents=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )						, ["hällo", "!", "how", "are", "you", "?"] )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )						, ["h\u00E9llo"] )






						def 					_lowerCAmelCase				(		self )       -> int:
								snake_case_ :							int         =				BasicTokenizer(do_lower_case=snake_case_						, strip_accents=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )						, ["hallo", "!", "how", "are", "you", "?"] )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )						, ["hello"] )






						def 					_lowerCAmelCase				(		self )       -> List[Any]:
								snake_case_ :							int         =				BasicTokenizer(do_lower_case=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )						, ["hallo", "!", "how", "are", "you", "?"] )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )						, ["hello"] )






						def 					_lowerCAmelCase				(		self )       -> Any:
								snake_case_ :							List[str]         =				BasicTokenizer(do_lower_case=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )						, ["HeLLo", "!", "how", "Are", "yoU", "?"] )






						def 					_lowerCAmelCase				(		self )       -> str:
								snake_case_ :							int         =				BasicTokenizer(do_lower_case=snake_case_						, strip_accents=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )						, ["HäLLo", "!", "how", "Are", "yoU", "?"] )






						def 					_lowerCAmelCase				(		self )       -> Optional[Any]:
								snake_case_ :							int         =				BasicTokenizer(do_lower_case=snake_case_						, strip_accents=snake_case_ )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )						, ["HaLLo", "!", "how", "Are", "yoU", "?"] )






						def 					_lowerCAmelCase				(		self )       -> Optional[Any]:
								snake_case_ :							Tuple         =				BasicTokenizer(do_lower_case=snake_case_						, never_split=["[UNK]"] )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]" )						, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] )






						def 					_lowerCAmelCase				(		self )       -> Tuple:
								snake_case_ :							List[Any]         =				['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

								snake_case_ :							str         =				{}
								for i, token in enumerate(snake_case_ ):
										snake_case_ :							int         =				i
								snake_case_ :							Any         =				WordpieceTokenizer(vocab=snake_case_						, unk_token="[UNK]" )

								self.assertListEqual(tokenizer.tokenize("" )						, [] )

								self.assertListEqual(tokenizer.tokenize("unwanted running" )						, ["un", "##want", "##ed", "runn", "##ing"] )

								self.assertListEqual(tokenizer.tokenize("unwantedX running" )						, ["[UNK]", "runn", "##ing"] )






						def 					_lowerCAmelCase				(		self )       -> Optional[int]:
								self.assertTrue(_is_whitespace(" " ) )
								self.assertTrue(_is_whitespace("\t" ) )
								self.assertTrue(_is_whitespace("\r" ) )
								self.assertTrue(_is_whitespace("\n" ) )
								self.assertTrue(_is_whitespace("\u00A0" ) )

								self.assertFalse(_is_whitespace("A" ) )
								self.assertFalse(_is_whitespace("-" ) )






						def 					_lowerCAmelCase				(		self )       -> Any:
								self.assertTrue(_is_control("\u0005" ) )

								self.assertFalse(_is_control("A" ) )
								self.assertFalse(_is_control(" " ) )
								self.assertFalse(_is_control("\t" ) )
								self.assertFalse(_is_control("\r" ) )






						def 					_lowerCAmelCase				(		self )       -> str:
								self.assertTrue(_is_punctuation("-" ) )
								self.assertTrue(_is_punctuation("$" ) )
								self.assertTrue(_is_punctuation("`" ) )
								self.assertTrue(_is_punctuation("." ) )

								self.assertFalse(_is_punctuation("A" ) )
								self.assertFalse(_is_punctuation(" " ) )






						def 					_lowerCAmelCase				(		self )       -> Tuple:
								snake_case_ :							Union[str, Any]         =				self.get_tokenizer()
								snake_case_ :							int         =				self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(snake_case_ ) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]] )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(snake_case_ ) for t in ["Test", "\xad", "test"]]						, [["[UNK]"], [], ["[UNK]"]] )






						@slow
						def 					_lowerCAmelCase				(		self )       -> int:
								snake_case_ :							Optional[Any]         =				self.tokenizer_class.from_pretrained("google/electra-base-discriminator" )

								snake_case_ :							List[str]         =				tokenizer.encode("sequence builders"						, add_special_tokens=snake_case_ )
								snake_case_ :							List[str]         =				tokenizer.encode("multi-sequence build"						, add_special_tokens=snake_case_ )

								snake_case_ :							Optional[int]         =				tokenizer.build_inputs_with_special_tokens(snake_case_ )
								snake_case_ :							Dict         =				tokenizer.build_inputs_with_special_tokens(snake_case_						, snake_case_ )

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]






						def 					_lowerCAmelCase				(		self )       -> int:

								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
												snake_case_ :							Union[str, Any]         =				self.rust_tokenizer_class.from_pretrained(snake_case_						, **snake_case_ )

												snake_case_ :							str         =				f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
												snake_case_ :							str         =				tokenizer_r.encode_plus(
												    snake_case_						, return_attention_mask=snake_case_						, return_token_type_ids=snake_case_						, return_offsets_mapping=snake_case_						, add_special_tokens=snake_case_						, )

												snake_case_ :							Optional[int]         =				tokenizer_r.do_lower_case if hasattr(snake_case_						, "do_lower_case" ) else False
												snake_case_ :							List[Any]         =				(
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'A'),
												        ((1, 2), ','),
												        ((3, 5), 'na'),
												        ((5, 6), '##ï'),
												        ((6, 8), '##ve'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'Allen'),
												        ((21, 23), '##NL'),
												        ((23, 24), '##P'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'a'),
												        ((1, 2), ','),
												        ((3, 8), 'naive'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'allen'),
												        ((21, 23), '##nl'),
												        ((23, 24), '##p'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]						, tokenizer_r.convert_ids_to_tokens(tokens["input_ids"] ) )
												self.assertEqual([e[0] for e in expected_results]						, tokens["offset_mapping"] )






						def 					_lowerCAmelCase				(		self )       -> Any:
								snake_case_ :							int         =				['的', '人', '有']
								snake_case_ :							Optional[int]         =				''.join(snake_case_ )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
												snake_case_ :							Tuple         =				True
												snake_case_ :							Union[str, Any]         =				self.tokenizer_class.from_pretrained(snake_case_						, **snake_case_ )
												snake_case_ :							List[Any]         =				self.rust_tokenizer_class.from_pretrained(snake_case_						, **snake_case_ )

												snake_case_ :							Union[str, Any]         =				tokenizer_p.encode(snake_case_						, add_special_tokens=snake_case_ )
												snake_case_ :							Any         =				tokenizer_r.encode(snake_case_						, add_special_tokens=snake_case_ )

												snake_case_ :							Any         =				tokenizer_r.convert_ids_to_tokens(snake_case_ )
												snake_case_ :							str         =				tokenizer_p.convert_ids_to_tokens(snake_case_ )

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(snake_case_						, snake_case_ )
												self.assertListEqual(snake_case_						, snake_case_ )

												snake_case_ :							Tuple         =				False
												snake_case_ :							Optional[Any]         =				self.rust_tokenizer_class.from_pretrained(snake_case_						, **snake_case_ )
												snake_case_ :							Union[str, Any]         =				self.tokenizer_class.from_pretrained(snake_case_						, **snake_case_ )

												snake_case_ :							str         =				tokenizer_r.encode(snake_case_						, add_special_tokens=snake_case_ )
												snake_case_ :							Tuple         =				tokenizer_p.encode(snake_case_						, add_special_tokens=snake_case_ )

												snake_case_ :							str         =				tokenizer_r.convert_ids_to_tokens(snake_case_ )
												snake_case_ :							Tuple         =				tokenizer_p.convert_ids_to_tokens(snake_case_ )

												# it is expected that only the first Chinese character is not preceded by "##".
												snake_case_ :							List[str]         =				[
												    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(snake_case_ )
												]
												self.assertListEqual(snake_case_						, snake_case_ )
												self.assertListEqual(snake_case_						, snake_case_ )

'''simple docstring'''


from __future__ import annotations

from collections import deque
from enum import Enum
from math import atana, degrees
from sys import maxsize






class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_       ):







  '''simple docstring'''






  lowerCamelCase_		:Union[str, Any]						    =		1
  lowerCamelCase_		:List[Any]						    =		2
  lowerCamelCase_		:Optional[Any]						    =		3


  def __repr__( self						):




       '''simple docstring'''

       return F'''{self.__class__.__name__}.{self.name}'''







def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							int	,     lowerCamelCase_					:							int   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   Any   =      point
     return degrees(atana(y - miny	,     x - minx   )   )


def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   str   =      starting
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Optional[int]   =      via
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Dict   =      target
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     via_angle %= 360
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     target_angle %= 360
     # t-
     #  \ \
     #   \ v
     #    \|
     #     s
     # via_angle is always lower than target_angle, if direction is left.
     # If they are same, it means they are on a same line of convex hull.
     if target_angle > via_angle:
          return Direction.left
     elif target_angle == via_angle:
          return Direction.straight
     else:
          return Direction.right





def 					_lowerCamelCase    (      lowerCamelCase_					:							list[tuple[int, int]]   ):



     """simple docstring"""





     if len(lowerCamelCase_   ) <= 2:
          # There is no convex hull
          raise ValueError('graham_scan: argument must contain more than 3 points.'   )
     if len(lowerCamelCase_   ) == 3:
          return points
     # find the lowest and the most left point
     UpperCAmelCase_							:   Dict   =      0
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Union[str, Any]   =      maxsize, maxsize
     for i, point in enumerate(lowerCamelCase_   ):
          UpperCAmelCase_							:   Union[str, Any]   =      point[0]
          UpperCAmelCase_							:   List[Any]   =      point[1]
          if y < miny:
               UpperCAmelCase_							:   Dict   =      y
               UpperCAmelCase_							:   Optional[int]   =      x
               UpperCAmelCase_							:   Optional[Any]   =      i
          if y == miny and x < minx:
               UpperCAmelCase_							:   Union[str, Any]   =      x
               UpperCAmelCase_							:   Dict   =      i

    # remove the lowest and the most left point from points for preparing for sort
     points.pop(lowerCamelCase_   )

     UpperCAmelCase_							:   List[str]   =      sorted(lowerCamelCase_	,     key=lambda lowerCamelCase_   : angle_comparer(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )   )
     # This insert actually costs complexity,
     # and you should instead add (minx, miny) into stack later.
     # I'm using insert just for easy understanding.
     sorted_points.insert(0	,     (minx, miny)   )

     UpperCAmelCase_							:   deque[tuple[int, int]]   =      deque()
     stack.append(sorted_points[0]   )
     stack.append(sorted_points[1]   )
     stack.append(sorted_points[2]   )
     # In any ways, the first 3 points line are towards left.
     # Because we sort them the angle from minx, miny.
     UpperCAmelCase_							:   List[Any]   =      Direction.left

     for i in range(3	,     len(lowerCamelCase_   )   ):
          while True:
               UpperCAmelCase_							:   Optional[Any]   =      stack[-2]
               UpperCAmelCase_							:   Optional[int]   =      stack[-1]
               UpperCAmelCase_							:   str   =      sorted_points[i]
               UpperCAmelCase_							:   Union[str, Any]   =      check_direction(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )

               if next_direction == Direction.left:
                    UpperCAmelCase_							:   Union[str, Any]   =      Direction.left
                    break
               if next_direction == Direction.straight:
                    if current_direction == Direction.left:
                         # We keep current_direction as left.
                         # Because if the straight line keeps as straight,
                         # we want to know if this straight line is towards left.
                         break
                    elif current_direction == Direction.right:
                         # If the straight line is towards right,
                         # every previous points on those straigh line is not convex hull.
                         stack.pop()
               if next_direction == Direction.right:
                    stack.pop()
          stack.append(sorted_points[i]   )
     return list(lowerCamelCase_   )






