'''simple docstring'''


import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__     :			str      			=  get_tests_dir('''fixtures/spiece.model''')






@require_sentencepiece
@require_tokenizers
class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_						,	unittest.TestCase       ):







  '''simple docstring'''






  lowerCamelCase_		:Union[str, Any]						    =		ErnieMTokenizer
  lowerCamelCase_		:Any						    =		False
  lowerCamelCase_		:List[str]						    =		True
  lowerCamelCase_		:str						    =		False
  lowerCamelCase_		:int						    =		False


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       super().setUp()

       # We have a SentencePiece fixture for testing
       UpperCAmelCase_							:   Any   =      ErnieMTokenizer(snake_case_      ,		unk_token='<unk>'      ,		pad_token='<pad>'						)
       tokenizer.save_pretrained(self.tmpdirname						)


  def _UpperCamelCase     ( self      ,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      'this is a test'
       UpperCAmelCase_							:   Any   =      'this is a test'
       return input_text, output_text


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      '<pad>'
       UpperCAmelCase_							:   Union[str, Any]   =      0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_						)      ,		snake_case_						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_						)      ,		snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Dict   =      list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]      ,		'<pad>'						)
       self.assertEqual(vocab_keys[1]      ,		'<unk>'						)
       self.assertEqual(vocab_keys[-1]      ,		'▁eloquent'						)
       self.assertEqual(len(snake_case_						)      ,		3_0_0_0_0						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       self.assertEqual(self.get_tokenizer().vocab_size      ,		3_0_0_0_0						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       if not self.test_rust_tokenizer:
            return

       UpperCAmelCase_							:   Any   =      self.get_tokenizer()
       UpperCAmelCase_							:   List[Any]   =      self.get_rust_tokenizer()

       UpperCAmelCase_							:   Union[str, Any]   =      'I was born in 92000, and this is falsé.'

       UpperCAmelCase_							:   Optional[Any]   =      tokenizer.tokenize(snake_case_						)
       UpperCAmelCase_							:   Tuple   =      rust_tokenizer.tokenize(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Optional[int]   =      tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       UpperCAmelCase_							:   str   =      rust_tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Optional[int]   =      self.get_rust_tokenizer()
       UpperCAmelCase_							:   Dict   =      tokenizer.encode(snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      rust_tokenizer.encode(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Any   =      ErnieMTokenizer(snake_case_      ,		do_lower_case=snake_case_      ,		unk_token='<unk>'      ,		pad_token='<pad>'						)

       UpperCAmelCase_							:   Optional[Any]   =      tokenizer.tokenize('This is a test'						)
       self.assertListEqual(snake_case_      ,		['▁this', '▁is', '▁a', '▁test']						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_						)      ,		[4_8, 2_5, 2_1, 1_2_8_9]						)

       UpperCAmelCase_							:   Tuple   =      tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
       self.assertListEqual(
           snake_case_      ,		['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']						)
       UpperCAmelCase_							:   Union[str, Any]   =      tokenizer.convert_tokens_to_ids(snake_case_						)
       self.assertListEqual(snake_case_      ,		[3_1, 2_3, 3_8_6, 1_9, 5_1_8, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]						)

       UpperCAmelCase_							:   Optional[Any]   =      tokenizer.convert_ids_to_tokens(snake_case_						)
       self.assertListEqual(
           snake_case_      ,		['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']      ,		)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      ErnieMTokenizer(snake_case_      ,		unk_token='<unk>'      ,		pad_token='<pad>'						)

       UpperCAmelCase_							:   Optional[int]   =      tokenizer.encode('sequence builders'						)
       UpperCAmelCase_							:   List[Any]   =      tokenizer.encode('multi-sequence build'						)

       UpperCAmelCase_							:   str   =      tokenizer.build_inputs_with_special_tokens(snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      tokenizer.build_inputs_with_special_tokens(snake_case_      ,		snake_case_						)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
           tokenizer.sep_token_id
       ] + text_a + [tokenizer.sep_token_id]






  @slow
  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      {'input_ids': [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 9, 3_0_4, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 5, 5, 5, 1_6, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 6_4_6_0, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 3_5_5_9, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=snake_case_      ,		model_name='susnato/ernie-m-base_pytorch'      ,		sequences=[
               'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
               'general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
               'Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained '
               'models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.',
               'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
               'conditioning on both left and right context in all layers.',
               'The quick brown fox jumps over the lazy dog.',
           ]      ,		)







'''simple docstring'''


from __future__ import annotations

from collections import deque
from enum import Enum
from math import atana, degrees
from sys import maxsize






class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_       ):







  '''simple docstring'''






  lowerCamelCase_		:Union[str, Any]						    =		1
  lowerCamelCase_		:List[Any]						    =		2
  lowerCamelCase_		:Optional[Any]						    =		3


  def __repr__( self						):




       '''simple docstring'''

       return F'''{self.__class__.__name__}.{self.name}'''







def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							int	,     lowerCamelCase_					:							int   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   Any   =      point
     return degrees(atana(y - miny	,     x - minx   )   )


def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   str   =      starting
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Optional[int]   =      via
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Dict   =      target
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     via_angle %= 360
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     target_angle %= 360
     # t-
     #  \ \
     #   \ v
     #    \|
     #     s
     # via_angle is always lower than target_angle, if direction is left.
     # If they are same, it means they are on a same line of convex hull.
     if target_angle > via_angle:
          return Direction.left
     elif target_angle == via_angle:
          return Direction.straight
     else:
          return Direction.right





def 					_lowerCamelCase    (      lowerCamelCase_					:							list[tuple[int, int]]   ):



     """simple docstring"""





     if len(lowerCamelCase_   ) <= 2:
          # There is no convex hull
          raise ValueError('graham_scan: argument must contain more than 3 points.'   )
     if len(lowerCamelCase_   ) == 3:
          return points
     # find the lowest and the most left point
     UpperCAmelCase_							:   Dict   =      0
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Union[str, Any]   =      maxsize, maxsize
     for i, point in enumerate(lowerCamelCase_   ):
          UpperCAmelCase_							:   Union[str, Any]   =      point[0]
          UpperCAmelCase_							:   List[Any]   =      point[1]
          if y < miny:
               UpperCAmelCase_							:   Dict   =      y
               UpperCAmelCase_							:   Optional[int]   =      x
               UpperCAmelCase_							:   Optional[Any]   =      i
          if y == miny and x < minx:
               UpperCAmelCase_							:   Union[str, Any]   =      x
               UpperCAmelCase_							:   Dict   =      i

    # remove the lowest and the most left point from points for preparing for sort
     points.pop(lowerCamelCase_   )

     UpperCAmelCase_							:   List[str]   =      sorted(lowerCamelCase_	,     key=lambda lowerCamelCase_   : angle_comparer(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )   )
     # This insert actually costs complexity,
     # and you should instead add (minx, miny) into stack later.
     # I'm using insert just for easy understanding.
     sorted_points.insert(0	,     (minx, miny)   )

     UpperCAmelCase_							:   deque[tuple[int, int]]   =      deque()
     stack.append(sorted_points[0]   )
     stack.append(sorted_points[1]   )
     stack.append(sorted_points[2]   )
     # In any ways, the first 3 points line are towards left.
     # Because we sort them the angle from minx, miny.
     UpperCAmelCase_							:   List[Any]   =      Direction.left

     for i in range(3	,     len(lowerCamelCase_   )   ):
          while True:
               UpperCAmelCase_							:   Optional[Any]   =      stack[-2]
               UpperCAmelCase_							:   Optional[int]   =      stack[-1]
               UpperCAmelCase_							:   str   =      sorted_points[i]
               UpperCAmelCase_							:   Union[str, Any]   =      check_direction(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )

               if next_direction == Direction.left:
                    UpperCAmelCase_							:   Union[str, Any]   =      Direction.left
                    break
               if next_direction == Direction.straight:
                    if current_direction == Direction.left:
                         # We keep current_direction as left.
                         # Because if the straight line keeps as straight,
                         # we want to know if this straight line is towards left.
                         break
                    elif current_direction == Direction.right:
                         # If the straight line is towards right,
                         # every previous points on those straigh line is not convex hull.
                         stack.pop()
               if next_direction == Direction.right:
                    stack.pop()
          stack.append(sorted_points[i]   )
     return list(lowerCamelCase_   )






