


"""simple docstring"""


import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				_SCREAMING_SNAKE_CASE				(							__UpperCAmelCase     ,   unittest.TestCase  ):
   """simple docstring"""



   _a   :       List[Any]									=    CodeGenTokenizer
   _a   :       Tuple									=    CodeGenTokenizerFast
   _a   :       Tuple									=    True
   _a   :       Dict									=    {'''add_prefix_space''': True}
   _a   :       int									=    False


   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    lowercase__       :       List[Any]				=						[
        """l""",
        """o""",
        """w""",
        """e""",
        """r""",
        """s""",
        """t""",
        """i""",
        """d""",
        """n""",
        """\u0120""",
        """\u0120l""",
        """\u0120n""",
        """\u0120lo""",
        """\u0120low""",
        """er""",
        """\u0120lowest""",
        """\u0120newer""",
        """\u0120wider""",
        """<unk>""",
        """<|endoftext|>""",
    ]
    lowercase__       :       Optional[Any]				=						dict(zip(lowerCamelCase__ ,     range(len(lowerCamelCase__						)						)						)						)
    lowercase__       :       Dict				=						["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
    lowercase__       :       str				=						{"""unk_token""": """<unk>"""}

    lowercase__       :       int				=						os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
    lowercase__       :       List[Any]				=						os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""merges_file"""]						)
    with open(self.vocab_file ,     """w""" ,     encoding="""utf-8"""						) as fp:
     fp.write(json.dumps(lowerCamelCase__						) + """\n"""						)
    with open(self.merges_file ,     """w""" ,     encoding="""utf-8"""						) as fp:
     fp.write("""\n""".join(lowerCamelCase__						)						)


   def 					UpperCAmelCase__(						self ,     **lowerCamelCase__						)      ->							Union[str, Any]:
    kwargs.update(self.special_tokens_map						)
    return CodeGenTokenizer.from_pretrained(self.tmpdirname ,     **lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     **lowerCamelCase__						)      ->							Any:
    kwargs.update(self.special_tokens_map						)
    return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,     **lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							int:
    lowercase__       :       Dict				=						"""lower newer"""
    lowercase__       :       Dict				=						"""lower newer"""
    return input_text, output_text


   def 					UpperCAmelCase__(						self						)      ->							int:
    lowercase__       :       Union[str, Any]				=						CodeGenTokenizer(self.vocab_file ,     self.merges_file ,     **self.special_tokens_map						)
    lowercase__       :       Optional[Any]				=						"""lower newer"""
    lowercase__       :       List[Any]				=						["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
    lowercase__       :       str				=						tokenizer.tokenize(lowerCamelCase__ ,     add_prefix_space=lowerCamelCase__						)
    self.assertListEqual(lowerCamelCase__ ,     lowerCamelCase__						)

    lowercase__       :       Tuple				=						tokens + [tokenizer.unk_token]
    lowercase__       :       int				=						[14, 15, 10, 9, 3, 2, 15, 19]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__						) ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							Union[str, Any]:
    if not self.test_rust_tokenizer:
     return

    lowercase__       :       Dict				=						self.get_tokenizer()
    lowercase__       :       List[str]				=						self.get_rust_tokenizer(add_prefix_space=lowerCamelCase__						)

    lowercase__       :       Union[str, Any]				=						"""lower newer"""

    # Testing tokenization
    lowercase__       :       int				=						tokenizer.tokenize(lowerCamelCase__ ,     add_prefix_space=lowerCamelCase__						)
    lowercase__       :       List[str]				=						rust_tokenizer.tokenize(lowerCamelCase__						)
    self.assertListEqual(lowerCamelCase__ ,     lowerCamelCase__						)

    # Testing conversion to ids without special tokens
    lowercase__       :       List[str]				=						tokenizer.encode(lowerCamelCase__ ,     add_special_tokens=lowerCamelCase__ ,     add_prefix_space=lowerCamelCase__						)
    lowercase__       :       Optional[Any]				=						rust_tokenizer.encode(lowerCamelCase__ ,     add_special_tokens=lowerCamelCase__						)
    self.assertListEqual(lowerCamelCase__ ,     lowerCamelCase__						)

    # Testing conversion to ids with special tokens
    lowercase__       :       List[Any]				=						self.get_rust_tokenizer(add_prefix_space=lowerCamelCase__						)
    lowercase__       :       Union[str, Any]				=						tokenizer.encode(lowerCamelCase__ ,     add_prefix_space=lowerCamelCase__						)
    lowercase__       :       List[str]				=						rust_tokenizer.encode(lowerCamelCase__						)
    self.assertListEqual(lowerCamelCase__ ,     lowerCamelCase__						)

    # Testing the unknown token
    lowercase__       :       int				=						tokens + [rust_tokenizer.unk_token]
    lowercase__       :       List[Any]				=						[14, 15, 10, 9, 3, 2, 15, 19]
    self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase__						) ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     *lowerCamelCase__ ,     **lowerCamelCase__						)      ->							Tuple:
    # It's very difficult to mix/test pretokenization with byte-level
    # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
    pass


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__=15						)      ->							Union[str, Any]:
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
      lowercase__       :       str				=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__ ,     **lowerCamelCase__						)

      # Simple input
      lowercase__       :       Optional[int]				=						"""This is a simple input"""
      lowercase__       :       int				=						["""This is a simple input 1""", """This is a simple input 2"""]
      lowercase__       :       Tuple				=						("""This is a simple input""", """This is a pair""")
      lowercase__       :       Optional[Any]				=						[
          ("""This is a simple input 1""", """This is a simple input 2"""),
          ("""This is a simple pair 1""", """This is a simple pair 2"""),
      ]

      # Simple input tests
      self.assertRaises(lowerCamelCase__ ,     tokenizer_r.encode ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length"""						)

      # Simple input
      self.assertRaises(lowerCamelCase__ ,     tokenizer_r.encode_plus ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length"""						)

      # Simple input
      self.assertRaises(
          lowerCamelCase__ ,     tokenizer_r.batch_encode_plus ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length""" ,     )

      # Pair input
      self.assertRaises(lowerCamelCase__ ,     tokenizer_r.encode ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length"""						)

      # Pair input
      self.assertRaises(lowerCamelCase__ ,     tokenizer_r.encode_plus ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length"""						)

      # Pair input
      self.assertRaises(
          lowerCamelCase__ ,     tokenizer_r.batch_encode_plus ,     lowerCamelCase__ ,     max_length=lowerCamelCase__ ,     padding="""max_length""" ,     )


   def 					UpperCAmelCase__(						self						)      ->							int:
    lowercase__       :       List[Any]				=						CodeGenTokenizer.from_pretrained(self.tmpdirname ,     pad_token="""<pad>"""						)

    # Simple input
    lowercase__       :       str				=						"""This is a simple input"""
    lowercase__       :       Tuple				=						["""This is a simple input looooooooong""", """This is a simple input"""]
    lowercase__       :       List[str]				=						("""This is a simple input""", """This is a pair""")
    lowercase__       :       Tuple				=						[
        ("""This is a simple input loooooong""", """This is a simple input"""),
        ("""This is a simple pair loooooong""", """This is a simple pair"""),
    ]

    lowercase__       :       Any				=						tokenizer.pad_token_id

    lowercase__       :       Tuple				=						tokenizer(lowerCamelCase__ ,     padding="""max_length""" ,     max_length=30 ,     return_tensors="""np"""						)
    lowercase__       :       int				=						tokenizer(lowerCamelCase__ ,     padding=lowerCamelCase__ ,     truncate=lowerCamelCase__ ,     return_tensors="""np"""						)
    lowercase__       :       List[Any]				=						tokenizer(*lowerCamelCase__ ,     padding="""max_length""" ,     max_length=60 ,     return_tensors="""np"""						)
    lowercase__       :       List[str]				=						tokenizer(lowerCamelCase__ ,     padding=lowerCamelCase__ ,     truncate=lowerCamelCase__ ,     return_tensors="""np"""						)

    # s
    # test single string max_length padding
    self.assertEqual(out_s["""input_ids"""].shape[-1] ,     30						)
    self.assertTrue(pad_token_id in out_s["""input_ids"""]						)
    self.assertTrue(0 in out_s["""attention_mask"""]						)

    # s2
    # test automatic padding
    self.assertEqual(out_sa["""input_ids"""].shape[-1] ,     33						)
    # long slice doesn't have padding
    self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]						)
    self.assertFalse(0 in out_sa["""attention_mask"""][0]						)
    # short slice does have padding
    self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]						)
    self.assertTrue(0 in out_sa["""attention_mask"""][1]						)

    # p
    # test single pair max_length padding
    self.assertEqual(out_p["""input_ids"""].shape[-1] ,     60						)
    self.assertTrue(pad_token_id in out_p["""input_ids"""]						)
    self.assertTrue(0 in out_p["""attention_mask"""]						)

    # p2
    # test automatic padding pair
    self.assertEqual(out_pa["""input_ids"""].shape[-1] ,     52						)
    # long slice pair doesn't have padding
    self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]						)
    self.assertFalse(0 in out_pa["""attention_mask"""][0]						)
    # short slice pair does have padding
    self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]						)
    self.assertTrue(0 in out_pa["""attention_mask"""][1]						)


   def 					UpperCAmelCase__(						self						)      ->							str:
    lowercase__       :       Any				=						"""$$$"""
    lowercase__       :       List[str]				=						CodeGenTokenizer.from_pretrained(self.tmpdirname ,     bos_token=lowerCamelCase__ ,     add_bos_token=lowerCamelCase__						)

    lowercase__       :       Optional[Any]				=						"""This is a simple input"""
    lowercase__       :       int				=						["""This is a simple input 1""", """This is a simple input 2"""]

    lowercase__       :       Optional[int]				=						tokenizer.bos_token_id

    lowercase__       :       Optional[Any]				=						tokenizer(lowerCamelCase__						)
    lowercase__       :       Any				=						tokenizer(lowerCamelCase__						)

    self.assertEqual(out_s.input_ids[0] ,     lowerCamelCase__						)
    self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

    lowercase__       :       List[Any]				=						tokenizer.decode(out_s.input_ids						)
    lowercase__       :       int				=						tokenizer.batch_decode(out_sa.input_ids						)

    self.assertEqual(decode_s.split()[0] ,     lowerCamelCase__						)
    self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)


   @slow
   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    lowercase__       :       Union[str, Any]				=						CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""						)

    lowercase__       :       Tuple				=						"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
    lowercase__       :       Optional[int]				=						"""\nif len_a > len_b:      result = a\nelse:      result = b"""

    lowercase__       :       Any				=						tokenizer.encode(lowerCamelCase__						)
    lowercase__       :       List[str]				=						["""^#""", re.escape("""<|endoftext|>"""						), """^'''""", """^\"\"\"""", """\n\n\n"""]
    lowercase__       :       Tuple				=						tokenizer.decode(lowerCamelCase__ ,     truncate_before_pattern=lowerCamelCase__						)
    self.assertEqual(lowerCamelCase__ ,     lowerCamelCase__						)







   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    pass



"""simple docstring"""


import requests
from bsa import BeautifulSoup


def    _lowerCamelCase   (     lowerCamelCase__	:							str = "https://www.worldometers.info/coronavirus"  ):
 lowercase__       :       Optional[Any]				=						BeautifulSoup(requests.get(lowerCamelCase__  ).text       ,		"""html.parser"""  )
 lowercase__       :       Union[str, Any]				=						soup.findAll("""h1"""  )
 lowercase__       :       List[str]				=						soup.findAll("""div"""       ,		{"""class""": """maincounter-number"""}  )
 keys += soup.findAll("""span"""       ,		{"""class""": """panel-title"""}  )
 values += soup.findAll("""div"""       ,		{"""class""": """number-table-main"""}  )
 return {key.text.strip(): value.text.strip() for key, value in zip(lowerCamelCase__       ,		lowerCamelCase__  )}


if __name__ == "__main__":
   print('\033[1m' + 'COVID-19 Status of the World' + '\033[0m\n')
   for key, value in world_covidaa_stats().items():
      print(F"{key}\n{value}\n")