'''simple docstring'''


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_						,	unittest.TestCase       ):







  '''simple docstring'''






  lowerCamelCase_		:Optional[int]						    =		ElectraTokenizer
  lowerCamelCase_		:Tuple						    =		ElectraTokenizerFast
  lowerCamelCase_		:List[Any]						    =		True
  lowerCamelCase_		:Optional[Any]						    =		True
  lowerCamelCase_		:int						    =		filter_non_english


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       super().setUp()

       UpperCAmelCase_							:   int   =      [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       UpperCAmelCase_							:   str   =      os.path.join(self.tmpdirname      ,		VOCAB_FILES_NAMES['vocab_file']						)
       with open(self.vocab_file      ,		'w'      ,		encoding='utf-8'						) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)


  def _UpperCamelCase     ( self      ,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[int]   =      'UNwant\u00E9d,running'
       UpperCAmelCase_							:   List[str]   =      'unwanted, running'
       return input_text, output_text


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[int]   =      self.tokenizer_class(self.vocab_file						)

       UpperCAmelCase_							:   int   =      tokenizer.tokenize('UNwant\u00E9d,running'						)
       self.assertListEqual(snake_case_      ,		['un', '##want', '##ed', ',', 'runn', '##ing']						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_						)      ,		[9, 6, 7, 1_2, 1_0, 1_1]						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       if not self.test_rust_tokenizer:
            return

       UpperCAmelCase_							:   Tuple   =      self.get_tokenizer()
       UpperCAmelCase_							:   Union[str, Any]   =      self.get_rust_tokenizer()

       UpperCAmelCase_							:   Tuple   =      'UNwant\u00E9d,running'

       UpperCAmelCase_							:   Dict   =      tokenizer.tokenize(snake_case_						)
       UpperCAmelCase_							:   Union[str, Any]   =      rust_tokenizer.tokenize(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Any   =      tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       UpperCAmelCase_							:   Dict   =      rust_tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   int   =      self.get_rust_tokenizer()
       UpperCAmelCase_							:   Dict   =      tokenizer.encode(snake_case_						)
       UpperCAmelCase_							:   List[str]   =      rust_tokenizer.encode(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       # With lower casing
       UpperCAmelCase_							:   Any   =      self.get_tokenizer(do_lower_case=snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      self.get_rust_tokenizer(do_lower_case=snake_case_						)

       UpperCAmelCase_							:   Tuple   =      'UNwant\u00E9d,running'

       UpperCAmelCase_							:   List[str]   =      tokenizer.tokenize(snake_case_						)
       UpperCAmelCase_							:   str   =      rust_tokenizer.tokenize(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Dict   =      tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       UpperCAmelCase_							:   Any   =      rust_tokenizer.encode(snake_case_      ,		add_special_tokens=snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)

       UpperCAmelCase_							:   Optional[int]   =      self.get_rust_tokenizer()
       UpperCAmelCase_							:   int   =      tokenizer.encode(snake_case_						)
       UpperCAmelCase_							:   Optional[int]   =      rust_tokenizer.encode(snake_case_						)
       self.assertListEqual(snake_case_      ,		snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)      ,		['ah', '\u535A', '\u63A8', 'zz']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[Any]   =      BasicTokenizer(do_lower_case=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)      ,		['hello', '!', 'how', 'are', 'you', '?']						)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,		['hello']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      BasicTokenizer(do_lower_case=snake_case_      ,		strip_accents=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,		['hällo', '!', 'how', 'are', 'you', '?']						)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,		['h\u00E9llo']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      BasicTokenizer(do_lower_case=snake_case_      ,		strip_accents=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,		['hallo', '!', 'how', 'are', 'you', '?']						)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,		['hello']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      BasicTokenizer(do_lower_case=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,		['hallo', '!', 'how', 'are', 'you', '?']						)
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,		['hello']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      BasicTokenizer(do_lower_case=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)      ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      BasicTokenizer(do_lower_case=snake_case_      ,		strip_accents=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,		['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      BasicTokenizer(do_lower_case=snake_case_      ,		strip_accents=snake_case_						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,		['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Tuple   =      BasicTokenizer(do_lower_case=snake_case_      ,		never_split=['[UNK]']						)

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)      ,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[Any]   =      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       UpperCAmelCase_							:   str   =      {}
       for i, token in enumerate(snake_case_						):
            UpperCAmelCase_							:   int   =      i
       UpperCAmelCase_							:   Any   =      WordpieceTokenizer(vocab=snake_case_      ,		unk_token='[UNK]'						)

       self.assertListEqual(tokenizer.tokenize(''						)      ,		[]						)

       self.assertListEqual(tokenizer.tokenize('unwanted running'						)      ,		['un', '##want', '##ed', 'runn', '##ing']						)

       self.assertListEqual(tokenizer.tokenize('unwantedX running'						)      ,		['[UNK]', 'runn', '##ing']						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       self.assertTrue(_is_whitespace(' '						)						)
       self.assertTrue(_is_whitespace('\t'						)						)
       self.assertTrue(_is_whitespace('\r'						)						)
       self.assertTrue(_is_whitespace('\n'						)						)
       self.assertTrue(_is_whitespace('\u00A0'						)						)

       self.assertFalse(_is_whitespace('A'						)						)
       self.assertFalse(_is_whitespace('-'						)						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       self.assertTrue(_is_control('\u0005'						)						)

       self.assertFalse(_is_control('A'						)						)
       self.assertFalse(_is_control(' '						)						)
       self.assertFalse(_is_control('\t'						)						)
       self.assertFalse(_is_control('\r'						)						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       self.assertTrue(_is_punctuation('-'						)						)
       self.assertTrue(_is_punctuation('$'						)						)
       self.assertTrue(_is_punctuation('`'						)						)
       self.assertTrue(_is_punctuation('.'						)						)

       self.assertFalse(_is_punctuation('A'						)						)
       self.assertFalse(_is_punctuation(' '						)						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      self.get_tokenizer()
       UpperCAmelCase_							:   int   =      self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(snake_case_						) for t in ['Test', '\xad', 'test']]      ,		[['[UNK]'], [], ['[UNK]']]						)

       self.assertListEqual(
           [rust_tokenizer.tokenize(snake_case_						) for t in ['Test', '\xad', 'test']]      ,		[['[UNK]'], [], ['[UNK]']]						)


  @slow
  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[Any]   =      self.tokenizer_class.from_pretrained('google/electra-base-discriminator'						)

       UpperCAmelCase_							:   List[str]   =      tokenizer.encode('sequence builders'      ,		add_special_tokens=snake_case_						)
       UpperCAmelCase_							:   List[str]   =      tokenizer.encode('multi-sequence build'      ,		add_special_tokens=snake_case_						)

       UpperCAmelCase_							:   Optional[int]   =      tokenizer.build_inputs_with_special_tokens(snake_case_						)
       UpperCAmelCase_							:   Dict   =      tokenizer.build_inputs_with_special_tokens(snake_case_      ,		snake_case_						)

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
                 UpperCAmelCase_							:   Union[str, Any]   =      self.rust_tokenizer_class.from_pretrained(snake_case_      ,		**snake_case_						)

                 UpperCAmelCase_							:   str   =      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                 UpperCAmelCase_							:   str   =      tokenizer_r.encode_plus(
                     snake_case_      ,		return_attention_mask=snake_case_      ,		return_token_type_ids=snake_case_      ,		return_offsets_mapping=snake_case_      ,		add_special_tokens=snake_case_      ,		)

                 UpperCAmelCase_							:   Optional[int]   =      tokenizer_r.do_lower_case if hasattr(snake_case_      ,		'do_lower_case'						) else False
                 UpperCAmelCase_							:   List[Any]   =      (
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'A'),
                         ((1, 2), ','),
                         ((3, 5), 'na'),
                         ((5, 6), '##ï'),
                         ((6, 8), '##ve'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'Allen'),
                         ((2_1, 2_3), '##NL'),
                         ((2_3, 2_4), '##P'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), 'a'),
                         ((1, 2), ','),
                         ((3, 8), 'naive'),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), 'allen'),
                         ((2_1, 2_3), '##nl'),
                         ((2_3, 2_4), '##p'),
                         ((2_5, 3_3), 'sentence'),
                         ((3_3, 3_4), '.'),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results]      ,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']						)						)
                 self.assertEqual([e[0] for e in expected_results]      ,		tokens['offset_mapping']						)






  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      ['的', '人', '有']
       UpperCAmelCase_							:   Optional[int]   =      ''.join(snake_case_						)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
                 UpperCAmelCase_							:   Tuple   =      True
                 UpperCAmelCase_							:   Union[str, Any]   =      self.tokenizer_class.from_pretrained(snake_case_      ,		**snake_case_						)
                 UpperCAmelCase_							:   List[Any]   =      self.rust_tokenizer_class.from_pretrained(snake_case_      ,		**snake_case_						)

                 UpperCAmelCase_							:   Union[str, Any]   =      tokenizer_p.encode(snake_case_      ,		add_special_tokens=snake_case_						)
                 UpperCAmelCase_							:   Any   =      tokenizer_r.encode(snake_case_      ,		add_special_tokens=snake_case_						)

                 UpperCAmelCase_							:   Any   =      tokenizer_r.convert_ids_to_tokens(snake_case_						)
                 UpperCAmelCase_							:   str   =      tokenizer_p.convert_ids_to_tokens(snake_case_						)

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(snake_case_      ,		snake_case_						)
                 self.assertListEqual(snake_case_      ,		snake_case_						)

                 UpperCAmelCase_							:   Tuple   =      False
                 UpperCAmelCase_							:   Optional[Any]   =      self.rust_tokenizer_class.from_pretrained(snake_case_      ,		**snake_case_						)
                 UpperCAmelCase_							:   Union[str, Any]   =      self.tokenizer_class.from_pretrained(snake_case_      ,		**snake_case_						)

                 UpperCAmelCase_							:   str   =      tokenizer_r.encode(snake_case_      ,		add_special_tokens=snake_case_						)
                 UpperCAmelCase_							:   Tuple   =      tokenizer_p.encode(snake_case_      ,		add_special_tokens=snake_case_						)

                 UpperCAmelCase_							:   str   =      tokenizer_r.convert_ids_to_tokens(snake_case_						)
                 UpperCAmelCase_							:   Tuple   =      tokenizer_p.convert_ids_to_tokens(snake_case_						)

                 # it is expected that only the first Chinese character is not preceded by "##".
                 UpperCAmelCase_							:   List[str]   =      [
                     F'''##{token}''' if idx != 0 else token for idx, token in enumerate(snake_case_						)
                 ]
                 self.assertListEqual(snake_case_      ,		snake_case_						)
                 self.assertListEqual(snake_case_      ,		snake_case_						)







'''simple docstring'''


def 					_lowerCamelCase    (      lowerCamelCase_					:							str   ):



     """simple docstring"""





     UpperCAmelCase_							:   str   =      str(lowerCamelCase_   ).strip()
     if not oct_string:
          raise ValueError('Empty string was passed to the function'   )
     UpperCAmelCase_							:   Tuple   =      oct_string[0] == '-'
     if is_negative:
          UpperCAmelCase_							:   Tuple   =      oct_string[1:]
     if not oct_string.isdigit() or not all(0 <= int(lowerCamelCase_   ) <= 7 for char in oct_string   ):
          raise ValueError('Non-octal value was passed to the function'   )
     UpperCAmelCase_							:   str   =      0
     for char in oct_string:
          UpperCAmelCase_							:   List[Any]   =      8 * decimal_number + int(lowerCamelCase_   )
     if is_negative:
          UpperCAmelCase_							:   List[Any]   =      -decimal_number
     return decimal_number


if __name__ == "__main__":
 from doctest import testmod

 testmod()






