






"""simple docstring"""







import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class      UpperCamelCase_       (				a_      ,    unittest.TestCase     ):
  _A				:    List[Any] =		RealmTokenizer
  _A				:    Union[str, Any] =		RealmTokenizerFast
  _A				:    Any =		True
  _A				:    Dict =		True
  _A				:    Union[str, Any] =		filter_non_english



  def 						UpperCamelCase_ (					self )	->	Tuple:



         """simple docstring"""





         super().setUp()

         UpperCAmelCase					=	[
             """[UNK]""",
             """[CLS]""",
             """[SEP]""",
             """[PAD]""",
             """[MASK]""",
             """want""",
             """##want""",
             """##ed""",
             """wa""",
             """un""",
             """runn""",
             """##ing""",
             """,""",
             """low""",
             """lowest""",
         ]
         UpperCAmelCase					=	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["""vocab_file"""] )
         with open(self.vocab_file       ,						"""w"""       ,						encoding="""utf-8""" ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )



  def 						UpperCamelCase_ (					self       ,						snake_case__ )	->	Optional[Any]:



         """simple docstring"""





         UpperCAmelCase					=	"""UNwant\u00E9d,running"""
         UpperCAmelCase					=	"""unwanted, running"""
         return input_text, output_text



  def 						UpperCamelCase_ (					self )	->	Any:



         """simple docstring"""





         UpperCAmelCase					=	self.tokenizer_class(self.vocab_file )

         UpperCAmelCase					=	tokenizer.tokenize("""UNwant\u00E9d,running""" )
         self.assertListEqual(snake_case__       ,						["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""] )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )       ,						[9, 6, 7, 12, 10, 11] )



  def 						UpperCamelCase_ (					self )	->	List[Any]:



         """simple docstring"""





         if not self.test_rust_tokenizer:
                return

         UpperCAmelCase					=	self.get_tokenizer()
         UpperCAmelCase					=	self.get_rust_tokenizer()

         UpperCAmelCase					=	"""UNwant\u00E9d,running"""

         UpperCAmelCase					=	tokenizer.tokenize(snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.tokenize(snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )

         UpperCAmelCase					=	tokenizer.encode(snake_case__       ,						add_special_tokens=snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.encode(snake_case__       ,						add_special_tokens=snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )

         UpperCAmelCase					=	self.get_rust_tokenizer()
         UpperCAmelCase					=	tokenizer.encode(snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.encode(snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )

         # With lower casing
         UpperCAmelCase					=	self.get_tokenizer(do_lower_case=snake_case__ )
         UpperCAmelCase					=	self.get_rust_tokenizer(do_lower_case=snake_case__ )

         UpperCAmelCase					=	"""UNwant\u00E9d,running"""

         UpperCAmelCase					=	tokenizer.tokenize(snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.tokenize(snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )

         UpperCAmelCase					=	tokenizer.encode(snake_case__       ,						add_special_tokens=snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.encode(snake_case__       ,						add_special_tokens=snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )

         UpperCAmelCase					=	self.get_rust_tokenizer()
         UpperCAmelCase					=	tokenizer.encode(snake_case__ )
         UpperCAmelCase					=	rust_tokenizer.encode(snake_case__ )
         self.assertListEqual(snake_case__       ,						snake_case__ )



  def 						UpperCamelCase_ (					self )	->	int:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )       ,						["""ah""", """\u535A""", """\u63A8""", """zz"""] )



  def 						UpperCamelCase_ (					self )	->	int:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )       ,						["""hello""", """!""", """how""", """are""", """you""", """?"""] )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,						["""hello"""] )



  def 						UpperCamelCase_ (					self )	->	Tuple:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__       ,						strip_accents=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,						["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,						["""h\u00E9llo"""] )



  def 						UpperCamelCase_ (					self )	->	Any:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__       ,						strip_accents=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,						["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,						["""hello"""] )



  def 						UpperCamelCase_ (					self )	->	Optional[Any]:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,						["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )       ,						["""hello"""] )



  def 						UpperCamelCase_ (					self )	->	Dict:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )       ,						["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )



  def 						UpperCamelCase_ (					self )	->	Dict:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__       ,						strip_accents=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,						["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )



  def 						UpperCamelCase_ (					self )	->	str:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__       ,						strip_accents=snake_case__ )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )       ,						["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )



  def 						UpperCamelCase_ (					self )	->	Dict:



         """simple docstring"""





         UpperCAmelCase					=	BasicTokenizer(do_lower_case=snake_case__       ,						never_split=["""[UNK]"""] )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )       ,						["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )



  def 						UpperCamelCase_ (					self )	->	List[Any]:



         """simple docstring"""





         UpperCAmelCase					=	["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

         UpperCAmelCase					=	{}
         for i, token in enumerate(snake_case__ ):
                UpperCAmelCase					=	i
         UpperCAmelCase					=	WordpieceTokenizer(vocab=snake_case__       ,						unk_token="""[UNK]""" )

         self.assertListEqual(tokenizer.tokenize("""""" )       ,						[] )

         self.assertListEqual(tokenizer.tokenize("""unwanted running""" )       ,						["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

         self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )       ,						["""[UNK]""", """runn""", """##ing"""] )



  def 						UpperCamelCase_ (					self )	->	Union[str, Any]:



         """simple docstring"""





         self.assertTrue(_is_whitespace(""" """ ) )
         self.assertTrue(_is_whitespace("""\t""" ) )
         self.assertTrue(_is_whitespace("""\r""" ) )
         self.assertTrue(_is_whitespace("""\n""" ) )
         self.assertTrue(_is_whitespace("""\u00A0""" ) )

         self.assertFalse(_is_whitespace("""A""" ) )
         self.assertFalse(_is_whitespace("""-""" ) )



  def 						UpperCamelCase_ (					self )	->	List[Any]:



         """simple docstring"""





         self.assertTrue(_is_control("""\u0005""" ) )

         self.assertFalse(_is_control("""A""" ) )
         self.assertFalse(_is_control(""" """ ) )
         self.assertFalse(_is_control("""\t""" ) )
         self.assertFalse(_is_control("""\r""" ) )



  def 						UpperCamelCase_ (					self )	->	List[str]:



         """simple docstring"""





         self.assertTrue(_is_punctuation("""-""" ) )
         self.assertTrue(_is_punctuation("""$""" ) )
         self.assertTrue(_is_punctuation("""`""" ) )
         self.assertTrue(_is_punctuation(""".""" ) )

         self.assertFalse(_is_punctuation("""A""" ) )
         self.assertFalse(_is_punctuation(""" """ ) )



  def 						UpperCamelCase_ (					self )	->	Union[str, Any]:



         """simple docstring"""





         UpperCAmelCase					=	self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(snake_case__ ) for t in ["""Test""", """\xad""", """test"""]]       ,						[["""[UNK]"""], [], ["""[UNK]"""]] )

         if self.test_rust_tokenizer:
                UpperCAmelCase					=	self.get_rust_tokenizer()
                self.assertListEqual(
                    [rust_tokenizer.tokenize(snake_case__ ) for t in ["""Test""", """\xad""", """test"""]]       ,						[["""[UNK]"""], [], ["""[UNK]"""]] )



  @slow
  def 						UpperCamelCase_ (					self )	->	Optional[Any]:



         """simple docstring"""





         UpperCAmelCase					=	self.tokenizer_class.from_pretrained("""bert-base-uncased""" )

         UpperCAmelCase					=	tokenizer.encode("""sequence builders"""       ,						add_special_tokens=snake_case__ )
         UpperCAmelCase					=	tokenizer.encode("""multi-sequence build"""       ,						add_special_tokens=snake_case__ )

         UpperCAmelCase					=	tokenizer.build_inputs_with_special_tokens(snake_case__ )
         UpperCAmelCase					=	tokenizer.build_inputs_with_special_tokens(snake_case__       ,						snake_case__ )

         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



  def 						UpperCamelCase_ (					self )	->	Optional[Any]:



         """simple docstring"""





         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                       UpperCAmelCase					=	self.rust_tokenizer_class.from_pretrained(snake_case__       ,						**snake_case__ )

                       UpperCAmelCase					=	f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                       UpperCAmelCase					=	tokenizer_r.encode_plus(
                           snake_case__       ,						return_attention_mask=snake_case__       ,						return_token_type_ids=snake_case__       ,						return_offsets_mapping=snake_case__       ,						add_special_tokens=snake_case__       ,						)

                       UpperCAmelCase					=	tokenizer_r.do_lower_case if hasattr(snake_case__       ,						"""do_lower_case""" ) else False
                       UpperCAmelCase					=	(
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """A"""),
                               ((1, 2), ""","""),
                               ((3, 5), """na"""),
                               ((5, 6), """##ï"""),
                               ((6, 8), """##ve"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """Allen"""),
                               ((21, 23), """##NL"""),
                               ((23, 24), """##P"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """a"""),
                               ((1, 2), ""","""),
                               ((3, 8), """naive"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """allen"""),
                               ((21, 23), """##nl"""),
                               ((23, 24), """##p"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
                       self.assertEqual([e[0] for e in expected_results]       ,						tokens["""offset_mapping"""] )



  @slow
  def 						UpperCamelCase_ (					self )	->	int:



         """simple docstring"""





         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                       UpperCAmelCase					=	self.rust_tokenizer_class.from_pretrained(snake_case__       ,						**snake_case__ )
                       UpperCAmelCase					=	self.tokenizer_class.from_pretrained(snake_case__       ,						**snake_case__ )
                       UpperCAmelCase					=	[["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

                       UpperCAmelCase					=	tokenizer_r.batch_encode_candidates(snake_case__       ,						max_length=10       ,						return_tensors="""np""" )
                       UpperCAmelCase					=	tokenizer_p.batch_encode_candidates(snake_case__       ,						max_length=10       ,						return_tensors="""np""" )

                       UpperCAmelCase					=	(2, 2, 10)

                       self.assertEqual(encoded_sentence_r["""input_ids"""].shape       ,						snake_case__ )
                       self.assertEqual(encoded_sentence_r["""attention_mask"""].shape       ,						snake_case__ )
                       self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape       ,						snake_case__ )

                       self.assertEqual(encoded_sentence_p["""input_ids"""].shape       ,						snake_case__ )
                       self.assertEqual(encoded_sentence_p["""attention_mask"""].shape       ,						snake_case__ )
                       self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape       ,						snake_case__ )








"""simple docstring"""







import unittest
from timeit import timeit






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	first_num if first_num >= second_num else second_num
       UpperCAmelCase					=	max_num
       while (common_mult % first_num > 0) or (common_mult % second_num > 0):
              common_mult += max_num
       return common_mult






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       return b if a == 0 else greatest_common_divisor(b % a		,					lowerCAmelCase						)






def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       return first_num // greatest_common_divisor(lowerCAmelCase		,					lowerCAmelCase						) * second_num






def 					_lowerCAmelCase				(					):







       '''simple docstring'''
       UpperCAmelCase					=	(
           """from __main__ import least_common_multiple_slow, least_common_multiple_fast"""
       )
       print(
           """least_common_multiple_slow():"""		,					timeit("""least_common_multiple_slow(1000, 999)"""		,					setup=lowerCAmelCase						)		,					)
       print(
           """least_common_multiple_fast():"""		,					timeit("""least_common_multiple_fast(1000, 999)"""		,					setup=lowerCAmelCase						)		,					)





class      UpperCamelCase_       (				unittest.TestCase     ):
  _A				:    List[str] =		(
      (10, 20),
      (13, 15),
      (4, 31),
      (10, 42),
      (43, 34),
      (5, 12),
      (12, 25),
      (10, 25),
      (6, 9),
  )
  _A				:    int =		(20, 195, 124, 210, 1462, 60, 300, 50, 18)



  def 						UpperCamelCase_ (					self )	->	Any:



         """simple docstring"""





         for i, (first_num, second_num) in enumerate(self.test_inputs ):
                UpperCAmelCase					=	least_common_multiple_slow(snake_case__       ,						snake_case__ )
                UpperCAmelCase					=	least_common_multiple_fast(snake_case__       ,						snake_case__ )
                with self.subTest(i=snake_case__ ):
                       self.assertEqual(snake_case__       ,						self.expected_results[i] )
                       self.assertEqual(snake_case__       ,						self.expected_results[i] )


if __name__ == "__main__":
 benchmark()
 unittest.main()
