






import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__       		=  """▁"""

lowerCamelCase__       		=  get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class       snake_case__ (				__snake_case						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					List[str]								=			BigBirdTokenizer
       lowerCamelCase				:					Dict								=			BigBirdTokenizerFast
       lowerCamelCase				:					int								=			True
       lowerCamelCase				:					Optional[Any]								=			True



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            super().setUp()

            __snake_case      :Optional[Any] 							=    self.tokenizer_class(__lowerCamelCase      ,    keep_accents=__lowerCamelCase     )
            tokenizer.save_pretrained(self.tmpdirname     )



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    """<s>"""
            __snake_case      :Dict 							=    1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase     )      ,    __lowerCamelCase     )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase     )      ,    __lowerCamelCase     )



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    list(self.get_tokenizer().get_vocab().keys()     )

            self.assertEqual(vocab_keys[0]      ,    """<unk>"""     )
            self.assertEqual(vocab_keys[1]      ,    """<s>"""     )
            self.assertEqual(vocab_keys[-1]      ,    """[MASK]"""     )
            self.assertEqual(len(__lowerCamelCase     )      ,    10_04     )



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            self.assertEqual(self.get_tokenizer().vocab_size      ,    10_00     )



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            if not self.test_rust_tokenizer:
                 return

            __snake_case      :Tuple 							=    self.get_tokenizer()
            __snake_case      :Tuple 							=    self.get_rust_tokenizer()

            __snake_case      :int 							=    """I was born in 92000, and this is falsé."""

            __snake_case      :str 							=    tokenizer.tokenize(__lowerCamelCase     )
            __snake_case      :Dict 							=    rust_tokenizer.tokenize(__lowerCamelCase     )
            self.assertListEqual(__lowerCamelCase      ,    __lowerCamelCase     )

            __snake_case      :Optional[int] 							=    tokenizer.encode(__lowerCamelCase      ,    add_special_tokens=__lowerCamelCase     )
            __snake_case      :Dict 							=    rust_tokenizer.encode(__lowerCamelCase      ,    add_special_tokens=__lowerCamelCase     )
            self.assertListEqual(__lowerCamelCase      ,    __lowerCamelCase     )

            __snake_case      :Optional[int] 							=    self.get_rust_tokenizer()
            __snake_case      :List[Any] 							=    tokenizer.encode(__lowerCamelCase     )
            __snake_case      :str 							=    rust_tokenizer.encode(__lowerCamelCase     )
            self.assertListEqual(__lowerCamelCase      ,    __lowerCamelCase     )



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Dict 							=    BigBirdTokenizer(__lowerCamelCase      ,    keep_accents=__lowerCamelCase     )

            __snake_case      :Union[str, Any] 							=    tokenizer.tokenize("""This is a test"""     )
            self.assertListEqual(__lowerCamelCase      ,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]     )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase     )      ,    [2_85, 46, 10, 1_70, 3_82]      ,    )

            __snake_case      :Tuple 							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
            self.assertListEqual(
                __lowerCamelCase      ,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]      ,    )
            __snake_case      :Any 							=    tokenizer.convert_tokens_to_ids(__lowerCamelCase     )
            self.assertListEqual(
                __lowerCamelCase      ,    [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,    )

            __snake_case      :int 							=    tokenizer.convert_ids_to_tokens(__lowerCamelCase     )
            self.assertListEqual(
                __lowerCamelCase      ,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]      ,    )



       @cached_property
       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""     )



       @slow
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :int 							=    """Hello World!"""
            __snake_case      :Dict 							=    [65, 1_85_36, 22_60, 1_01, 66]

            self.assertListEqual(__lowerCamelCase      ,    self.big_tokenizer.encode(__lowerCamelCase     )     )



       @slow
       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Dict 							=    (
                """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
                """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
            )
            # fmt: off
            __snake_case      :Tuple 							=    [65, 8_71, 4_19, 3_58, 9_46, 9_91, 25_21, 4_52, 3_58, 13_57, 3_87, 77_51, 35_36, 1_12, 9_85, 4_56, 1_26, 8_65, 9_38, 54_00, 57_34, 4_58, 13_68, 4_67, 7_86, 24_62, 52_46, 11_59, 6_33, 8_65, 45_19, 4_57, 5_82, 8_52, 25_57, 4_27, 9_16, 5_08, 4_05, 3_43_24, 4_97, 3_91, 4_08, 1_13_42, 12_44, 3_85, 1_00, 9_38, 9_85, 4_56, 5_74, 3_62, 1_25_97, 32_00, 31_29, 11_72, 66]  # noqa: E231
            # fmt: on
            self.assertListEqual(__lowerCamelCase      ,    self.big_tokenizer.encode(__lowerCamelCase     )     )



       @require_torch
       @slow
       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            import torch

            from transformers import BigBirdConfig, BigBirdModel

            # Build sequence
            __snake_case      :str 							=    list(self.big_tokenizer.get_vocab().keys()     )[:10]
            __snake_case      :Tuple 							=    """ """.join(__lowerCamelCase     )
            __snake_case      :Tuple 							=    self.big_tokenizer.encode_plus(__lowerCamelCase      ,    return_tensors="""pt"""      ,    return_token_type_ids=__lowerCamelCase     )
            __snake_case      :List[Any] 							=    self.big_tokenizer.batch_encode_plus(
                [sequence + """ """ + sequence]      ,    return_tensors="""pt"""      ,    return_token_type_ids=__lowerCamelCase     )

            __snake_case      :Tuple 							=    BigBirdConfig(attention_type="""original_full"""     )
            __snake_case      :Tuple 							=    BigBirdModel(__lowerCamelCase     )

            assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

            with torch.no_grad():
                 model(**__lowerCamelCase     )
                 model(**__lowerCamelCase     )



       @slow
       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            __snake_case      :Tuple 							=    BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""     )
            __snake_case      :str 							=    tokenizer.decode(tokenizer("""Paris is the [MASK]."""     ).input_ids     )

            self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""     )



       @slow
       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    {"""input_ids""": [[65, 3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14, 66], [65, 4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=__lowerCamelCase      ,    model_name="""google/bigbird-roberta-base"""      ,    revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""      ,    )






import argparse
import os
import re

import packaging.version


__a	 =      """examples/"""
__a	 =      {
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
__a	 =      {
    """init""": """src/diffusers/__init__.py""",
    """setup""": """setup.py""",
}
__a	 =      """README.md"""
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[Any]:
   with open(lowerCAmelCase_	,    """r"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      UpperCAmelCase      					=  f.read()
   UpperCAmelCase					,	UpperCAmelCase      					=  REPLACE_PATTERNS[pattern]
   UpperCAmelCase      					=  replace.replace("""VERSION"""	,    lowerCAmelCase_ )
   UpperCAmelCase      					=  re_pattern.sub(lowerCAmelCase_	,    lowerCAmelCase_ )
   with open(lowerCAmelCase_	,    """w"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      f.write(lowerCAmelCase_ )
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->int:
   for folder, directories, fnames in os.walk(lowerCAmelCase_ ):
      # Removing some of the folders with non-actively maintained examples from the walk
      if "research_projects" in directories:
         directories.remove("""research_projects""" )
      if "legacy" in directories:
         directories.remove("""legacy""" )
      for fname in fnames:
         if fname.endswith(""".py""" ):
            update_version_in_file(os.path.join(lowerCAmelCase_	,    lowerCAmelCase_ )	,    lowerCAmelCase_	,    pattern="""examples""" )
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_=False )    ->Optional[int]:
   for pattern, fname in REPLACE_FILES.items():
      update_version_in_file(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )
   if not patch:
      update_version_in_examples(lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->List[str]:
   UpperCAmelCase      					=  """🤗 Transformers currently provides the following architectures"""
   UpperCAmelCase      					=  """1. Want to contribute a new model?"""
   with open(lowerCAmelCase_	,    """r"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      UpperCAmelCase      					=  f.readlines()

   # Find the start of the list.
   UpperCAmelCase      					=  0
   while not lines[start_index].startswith(_start_prompt ):
      start_index += 1
   start_index += 1

   UpperCAmelCase      					=  start_index
   # Update the lines in the model list.
   while not lines[index].startswith(_end_prompt ):
      if lines[index].startswith("""1.""" ):
         UpperCAmelCase      					=  lines[index].replace(
             """https://huggingface.co/docs/diffusers/main/model_doc"""	,    """https://huggingface.co/docs/diffusers/model_doc"""	,    )
      index += 1

   with open(lowerCAmelCase_	,    """w"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      f.writelines(lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->List[str]:
   with open(REPLACE_FILES["""init"""]	,    """r""" ) as f:
      UpperCAmelCase      					=  f.read()
   UpperCAmelCase      					=  REPLACE_PATTERNS["""init"""][0].search(lowerCAmelCase_ ).groups()[0]
   return packaging.version.parse(lowerCAmelCase_ )
def 	_UpperCamelCase					(    lowerCAmelCase_=False )    ->int:
   UpperCAmelCase      					=  get_version()
   if patch and default_version.is_devrelease:
      raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!""" )
   if default_version.is_devrelease:
      UpperCAmelCase      					=  default_version.base_version
   elif patch:
      UpperCAmelCase      					=  F"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
   else:
      UpperCAmelCase      					=  F"""{default_version.major}.{default_version.minor + 1}.0"""

   # Now let's ask nicely if that's the right one.
   UpperCAmelCase      					=  input(F"""Which version are you releasing? [{default_version}]""" )
   if len(lowerCAmelCase_ ) == 0:
      UpperCAmelCase      					=  default_version

   print(F"""Updating version to {version}.""" )
   global_version_update(lowerCAmelCase_	,    patch=lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->int:
   UpperCAmelCase      					=  get_version()
   UpperCAmelCase      					=  F"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
   UpperCAmelCase      					=  current_version.base_version

   # Check with the user we got that right.
   UpperCAmelCase      					=  input(F"""Which version are we developing now? [{dev_version}]""" )
   if len(lowerCAmelCase_ ) == 0:
      UpperCAmelCase      					=  dev_version

   print(F"""Updating version to {version}.""" )
   global_version_update(lowerCAmelCase_ )


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
 __a	 =      argparse.ArgumentParser()
 parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
 parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
 __a	 =      parser.parse_args()
 if not args.post_release:
  pre_release_work(patch=args.patch)
 elif args.patch:
  print("""Nothing to do after a patch :-)""")
 else:
  post_release_work()



