


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class lowerCAmelCase_						( lowerCamelCase__					,						unittest.TestCase       ):

 '''simple docstring'''

 __snake_case												=  RealmTokenizer
 __snake_case												=  RealmTokenizerFast
 __snake_case												=  True
 __snake_case												=  True
 __snake_case												=  filter_non_english



 def 						UpperCamelCase__     (	self					):
       super().setUp()

       snake_case_       =      [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       snake_case_       =      os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES['''vocab_file''']					)
       with open(self.vocab_file		,      '''w'''		,      encoding='''utf-8'''					) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)



 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase					):
       snake_case_       =      '''UNwant\u00E9d,running'''
       snake_case_       =      '''unwanted, running'''
       return input_text, output_text



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      self.tokenizer_class(self.vocab_file					)

       snake_case_       =      tokenizer.tokenize('''UNwant\u00E9d,running'''					)
       self.assertListEqual(_UpperCAmelCase		,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)		,      [9, 6, 7, 12, 10, 11]					)



 def 						UpperCamelCase__     (	self					):
       if not self.test_rust_tokenizer:
             return

       snake_case_       =      self.get_tokenizer()
       snake_case_       =      self.get_rust_tokenizer()

       snake_case_       =      '''UNwant\u00E9d,running'''

       snake_case_       =      tokenizer.tokenize(_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.tokenize(_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       snake_case_       =      tokenizer.encode(_UpperCAmelCase		,      add_special_tokens=_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.encode(_UpperCAmelCase		,      add_special_tokens=_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       snake_case_       =      self.get_rust_tokenizer()
       snake_case_       =      tokenizer.encode(_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.encode(_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       # With lower casing
       snake_case_       =      self.get_tokenizer(do_lower_case=_UpperCAmelCase					)
       snake_case_       =      self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase					)

       snake_case_       =      '''UNwant\u00E9d,running'''

       snake_case_       =      tokenizer.tokenize(_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.tokenize(_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       snake_case_       =      tokenizer.encode(_UpperCAmelCase		,      add_special_tokens=_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.encode(_UpperCAmelCase		,      add_special_tokens=_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)

       snake_case_       =      self.get_rust_tokenizer()
       snake_case_       =      tokenizer.encode(_UpperCAmelCase					)
       snake_case_       =      rust_tokenizer.encode(_UpperCAmelCase					)
       self.assertListEqual(_UpperCAmelCase		,      _UpperCAmelCase					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)		,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)		,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,      ['''hello''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase		,      strip_accents=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,      ['''h\u00E9llo''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase		,      strip_accents=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,      ['''hello''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)		,      ['''hello''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase		,      strip_accents=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase		,      strip_accents=_UpperCAmelCase					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)		,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      BasicTokenizer(do_lower_case=_UpperCAmelCase		,      never_split=['''[UNK]''']					)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)		,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       snake_case_       =      {}
       for i, token in enumerate(_UpperCAmelCase					):
             snake_case_       =      i
       snake_case_       =      WordpieceTokenizer(vocab=_UpperCAmelCase		,      unk_token='''[UNK]'''					)

       self.assertListEqual(tokenizer.tokenize(''''''					)		,      []					)

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)		,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)		,      ['''[UNK]''', '''runn''', '''##ing''']					)



 def 						UpperCamelCase__     (	self					):
       self.assertTrue(_is_whitespace(''' '''					)					)
       self.assertTrue(_is_whitespace('''\t'''					)					)
       self.assertTrue(_is_whitespace('''\r'''					)					)
       self.assertTrue(_is_whitespace('''\n'''					)					)
       self.assertTrue(_is_whitespace('''\u00A0'''					)					)

       self.assertFalse(_is_whitespace('''A'''					)					)
       self.assertFalse(_is_whitespace('''-'''					)					)



 def 						UpperCamelCase__     (	self					):
       self.assertTrue(_is_control('''\u0005'''					)					)

       self.assertFalse(_is_control('''A'''					)					)
       self.assertFalse(_is_control(''' '''					)					)
       self.assertFalse(_is_control('''\t'''					)					)
       self.assertFalse(_is_control('''\r'''					)					)



 def 						UpperCamelCase__     (	self					):
       self.assertTrue(_is_punctuation('''-'''					)					)
       self.assertTrue(_is_punctuation('''$'''					)					)
       self.assertTrue(_is_punctuation('''`'''					)					)
       self.assertTrue(_is_punctuation('''.'''					)					)

       self.assertFalse(_is_punctuation('''A'''					)					)
       self.assertFalse(_is_punctuation(''' '''					)					)



 def 						UpperCamelCase__     (	self					):
       snake_case_       =      self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase					) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]					)

       if self.test_rust_tokenizer:
             snake_case_       =      self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(_UpperCAmelCase					) for t in ['''Test''', '''\xad''', '''test''']]		,      [['''[UNK]'''], [], ['''[UNK]''']]					)



 @slow
 def 						UpperCamelCase__     (	self					):
       snake_case_       =      self.tokenizer_class.from_pretrained('''bert-base-uncased'''					)

       snake_case_       =      tokenizer.encode('''sequence builders'''		,      add_special_tokens=_UpperCAmelCase					)
       snake_case_       =      tokenizer.encode('''multi-sequence build'''		,      add_special_tokens=_UpperCAmelCase					)

       snake_case_       =      tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					)
       snake_case_       =      tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase		,      _UpperCAmelCase					)

       assert encoded_sentence == [1_01] + text + [1_02]
       assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



 def 						UpperCamelCase__     (	self					):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   snake_case_       =      self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase		,      **_UpperCAmelCase					)

                   snake_case_       =      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   snake_case_       =      tokenizer_r.encode_plus(
                       _UpperCAmelCase		,      return_attention_mask=_UpperCAmelCase		,      return_token_type_ids=_UpperCAmelCase		,      return_offsets_mapping=_UpperCAmelCase		,      add_special_tokens=_UpperCAmelCase		,      )

                   snake_case_       =      tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase		,      '''do_lower_case'''					) else False
                   snake_case_       =      (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''Allen'''),
                           ((21, 23), '''##NL'''),
                           ((23, 24), '''##P'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), '''allen'''),
                           ((21, 23), '''##nl'''),
                           ((23, 24), '''##p'''),
                           ((25, 33), '''sentence'''),
                           ((33, 34), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]		,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
                   self.assertEqual([e[0] for e in expected_results]		,      tokens['''offset_mapping''']					)





 @slow
 def 						UpperCamelCase__     (	self					):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   snake_case_       =      self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase		,      **_UpperCAmelCase					)
                   snake_case_       =      self.tokenizer_class.from_pretrained(_UpperCAmelCase		,      **_UpperCAmelCase					)
                   snake_case_       =      [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

                   snake_case_       =      tokenizer_r.batch_encode_candidates(_UpperCAmelCase		,      max_length=10		,      return_tensors='''np'''					)
                   snake_case_       =      tokenizer_p.batch_encode_candidates(_UpperCAmelCase		,      max_length=10		,      return_tensors='''np'''					)

                   snake_case_       =      (2, 2, 10)

                   self.assertEqual(encoded_sentence_r['''input_ids'''].shape		,      _UpperCAmelCase					)
                   self.assertEqual(encoded_sentence_r['''attention_mask'''].shape		,      _UpperCAmelCase					)
                   self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape		,      _UpperCAmelCase					)

                   self.assertEqual(encoded_sentence_p['''input_ids'''].shape		,      _UpperCAmelCase					)
                   self.assertEqual(encoded_sentence_p['''attention_mask'''].shape		,      _UpperCAmelCase					)
                   self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape		,      _UpperCAmelCase					)



def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  int:




      """simple docstring"""

      if number != int(SCREAMING_SNAKE_CASE					):
            raise ValueError('''factorial() only accepts integral values'''					)
      if number < 0:
            raise ValueError('''factorial() not defined for negative values'''					)
      snake_case_       =      1
      for i in range(1					,					number + 1					):
            value *= i
      return value






def        __lowerCAmelCase    (SCREAMING_SNAKE_CASE					)->  int:




      """simple docstring"""

      if not isinstance(SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE					):
            raise ValueError('''factorial() only accepts integral values'''					)
      if n < 0:
            raise ValueError('''factorial() not defined for negative values'''					)
      return 1 if n == 0 or n == 1 else n * factorial(n - 1					)


if __name__ == "__main__":
   import doctest

   doctest.testmod()

   UpperCAmelCase			=    int(input("""Enter a positive integer: """).strip() or 0)
   print(f'''factorial{n} is {factorial(n)}''')