


import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__a	 =      get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")






@require_sentencepiece
@require_tokenizers
class 	__lowercase					(	__snake_case			,       unittest.TestCase	):
      UpperCamelCase										=							PegasusTokenizer
      UpperCamelCase										=							PegasusTokenizerFast
      UpperCamelCase										=							True
      UpperCamelCase										=							True





      def 							_lowercase					(		self    :       List[Any]							)     -> Tuple:






         """simple docstring"""







         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase      					=  PegasusTokenizer(__lowerCamelCase							)
         tokenizer.save_pretrained(self.tmpdirname							)





      @cached_property
      def 							_lowercase					(		self    :       Optional[Any]							)     -> int:






         """simple docstring"""







         return PegasusTokenizer.from_pretrained("""google/pegasus-large"""							)





      def 							_lowercase					(		self    :       Optional[Any]       ,							**__lowerCamelCase    :       Any							)     -> PegasusTokenizer:






         """simple docstring"""







         return PegasusTokenizer.from_pretrained(self.tmpdirname       ,							**__lowerCamelCase							)





      def 							_lowercase					(		self    :       Dict       ,							__lowerCamelCase    :       str							)     -> Optional[int]:






         """simple docstring"""







         return ("This is a test", "This is a test")





      def 							_lowercase					(		self    :       Tuple							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  """</s>"""
         UpperCAmelCase      					=  1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase							)       ,							__lowerCamelCase							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase							)       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]       ,							"""<pad>"""							)
         self.assertEqual(vocab_keys[1]       ,							"""</s>"""							)
         self.assertEqual(vocab_keys[-1]       ,							"""v"""							)
         self.assertEqual(len(__lowerCamelCase							)       ,							1_1_0_3							)





      def 							_lowercase					(		self    :       int							)     -> Union[str, Any]:






         """simple docstring"""







         self.assertEqual(self.get_tokenizer().vocab_size       ,							1_1_0_3							)





      def 							_lowercase					(		self    :       str							)     -> List[Any]:






         """simple docstring"""







         UpperCAmelCase      					=  self.rust_tokenizer_class.from_pretrained(self.tmpdirname							)
         UpperCAmelCase      					=  self.tokenizer_class.from_pretrained(self.tmpdirname							)
         UpperCAmelCase      					=  (
             """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
             """ </s> <pad> <pad> <pad>"""
         )
         UpperCAmelCase      					=  rust_tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase       ,							add_special_tokens=__lowerCamelCase							).input_ids[0]
         UpperCAmelCase      					=  py_tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase       ,							add_special_tokens=__lowerCamelCase							).input_ids[0]
         self.assertListEqual(__lowerCamelCase       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       Tuple							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  self._large_tokenizer
         # <mask_1> masks whole sentence while <mask_2> masks single word
         UpperCAmelCase      					=  """<mask_1> To ensure a <mask_2> flow of bank resolutions."""
         UpperCAmelCase      					=  [2, 4_1_3, 6_1_5, 1_1_4, 3, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
         UpperCAmelCase      					=  tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase							).input_ids[0]
         self.assertListEqual(__lowerCamelCase       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       Dict							)     -> List[str]:






         """simple docstring"""







         UpperCAmelCase      					=  self._large_tokenizer
         # The tracebacks for the following asserts are **better** without messages or self.assertEqual
         assert tokenizer.vocab_size == 9_6_1_0_3
         assert tokenizer.pad_token_id == 0
         assert tokenizer.eos_token_id == 1
         assert tokenizer.offset == 1_0_3
         assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_0_5
         assert tokenizer.unk_token == "<unk>"
         assert tokenizer.model_max_length == 1_0_2_4
         UpperCAmelCase      					=  """To ensure a smooth flow of bank resolutions."""
         UpperCAmelCase      					=  [4_1_3, 6_1_5, 1_1_4, 2_2_9_1, 1_9_7_1, 1_1_3, 1_6_7_9, 1_0_7_1_0, 1_0_7, 1]
         UpperCAmelCase      					=  tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase							).input_ids[0]
         self.assertListEqual(__lowerCamelCase       ,							__lowerCamelCase							)
         assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]							) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]





      @require_torch
      def 							_lowercase					(		self    :       Any							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  ["""This is going to be way too long.""" * 1_5_0, """short example"""]
         UpperCAmelCase      					=  ["""not super long but more than 5 tokens""", """tiny"""]
         UpperCAmelCase      					=  self._large_tokenizer(__lowerCamelCase       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							return_tensors="""pt"""							)
         UpperCAmelCase      					=  self._large_tokenizer(
             text_target=__lowerCamelCase       ,							max_length=5       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							return_tensors="""pt"""							)

         assert batch.input_ids.shape == (2, 1_0_2_4)
         assert batch.attention_mask.shape == (2, 1_0_2_4)
         assert targets["input_ids"].shape == (2, 5)
         assert len(__lowerCamelCase							) == 2  # input_ids, attention_mask.





      @slow
      def 							_lowercase					(		self    :       int							)     -> List[str]:






         """simple docstring"""







         UpperCAmelCase      					=  {"""input_ids""": [[3_8_9_7_9, 1_4_3, 1_8_4_8_5, 6_0_6, 1_3_0, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 5_4_1_8_9, 1_1_2_9, 1_1_1, 2_6_6_6_9, 8_7_6_8_6, 1_2_1, 9_1_1_4, 1_4_7_8_7, 1_2_1, 1_3_2_4_9, 1_5_8, 5_9_2, 9_5_6, 1_2_1, 1_4_6_2_1, 3_1_5_7_6, 1_4_3, 6_2_6_1_3, 1_0_8, 9_6_8_8, 9_3_0, 4_3_4_3_0, 1_1_5_6_2, 6_2_6_1_3, 3_0_4, 1_0_8, 1_1_4_4_3, 8_9_7, 1_0_8, 9_3_1_4, 1_7_4_1_5, 6_3_3_9_9, 1_0_8, 1_1_4_4_3, 7_6_1_4, 1_8_3_1_6, 1_1_8, 4_2_8_4, 7_1_4_8, 1_2_4_3_0, 1_4_3, 1_4_0_0, 2_5_7_0_3, 1_5_8, 1_1_1, 4_2_8_4, 7_1_4_8, 1_1_7_7_2, 1_4_3, 2_1_2_9_7, 1_0_6_4, 1_5_8, 1_2_2, 2_0_4, 3_5_0_6, 1_7_5_4, 1_1_3_3, 1_4_7_8_7, 1_5_8_1, 1_1_5, 3_3_2_2_4, 4_4_8_2, 1_1_1, 1_3_5_5, 1_1_0, 2_9_1_7_3, 3_1_7, 5_0_8_3_3, 1_0_8, 2_0_1_4_7, 9_4_6_6_5, 1_1_1, 7_7_1_9_8, 1_0_7, 1], [1_1_0, 6_2_6_1_3, 1_1_7, 6_3_8, 1_1_2, 1_1_3_3, 1_2_1, 2_0_0_9_8, 1_3_5_5, 7_9_0_5_0, 1_3_8_7_2, 1_3_5, 1_5_9_6, 5_3_5_4_1, 1_3_5_2, 1_4_1, 1_3_0_3_9, 5_5_4_2, 1_2_4, 3_0_2, 5_1_8, 1_1_1, 2_6_8, 2_9_5_6, 1_1_5, 1_4_9, 4_4_2_7, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_3_9, 1_2_3_5, 2_7_9_9, 1_8_2_8_9, 1_7_7_8_0, 2_0_4, 1_0_9, 9_4_7_4, 1_2_9_6, 1_0_7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__lowerCamelCase       ,							model_name="""google/bigbird-pegasus-large-arxiv"""       ,							revision="""ba85d0851d708441f91440d509690f1ab6353415"""       ,							)






@require_sentencepiece
@require_tokenizers
class 	__lowercase					(	__snake_case			,       unittest.TestCase	):
      UpperCamelCase										=							PegasusTokenizer
      UpperCamelCase										=							PegasusTokenizerFast
      UpperCamelCase										=							True
      UpperCamelCase										=							True





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> Tuple:






         """simple docstring"""







         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase      					=  PegasusTokenizer(__lowerCamelCase       ,							offset=0       ,							mask_token_sent=__lowerCamelCase       ,							mask_token="""[MASK]"""							)
         tokenizer.save_pretrained(self.tmpdirname							)





      @cached_property
      def 							_lowercase					(		self    :       Union[str, Any]							)     -> str:






         """simple docstring"""







         return PegasusTokenizer.from_pretrained("""google/bigbird-pegasus-large-arxiv"""							)





      def 							_lowercase					(		self    :       Any       ,							**__lowerCamelCase    :       Any							)     -> PegasusTokenizer:






         """simple docstring"""







         return PegasusTokenizer.from_pretrained(self.tmpdirname       ,							**__lowerCamelCase							)





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Union[str, Any]							)     -> List[str]:






         """simple docstring"""







         return ("This is a test", "This is a test")





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  self.rust_tokenizer_class.from_pretrained(self.tmpdirname							)
         UpperCAmelCase      					=  self.tokenizer_class.from_pretrained(self.tmpdirname							)
         UpperCAmelCase      					=  (
             """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
             """ <pad> <pad> <pad>"""
         )
         UpperCAmelCase      					=  rust_tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase       ,							add_special_tokens=__lowerCamelCase							).input_ids[0]
         UpperCAmelCase      					=  py_tokenizer([raw_input_str]       ,							return_tensors=__lowerCamelCase       ,							add_special_tokens=__lowerCamelCase							).input_ids[0]
         self.assertListEqual(__lowerCamelCase       ,							__lowerCamelCase							)





      @require_torch
      def 							_lowercase					(		self    :       List[Any]							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  ["""This is going to be way too long.""" * 1_0_0_0, """short example"""]
         UpperCAmelCase      					=  ["""not super long but more than 5 tokens""", """tiny"""]
         UpperCAmelCase      					=  self._large_tokenizer(__lowerCamelCase       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							return_tensors="""pt"""							)
         UpperCAmelCase      					=  self._large_tokenizer(
             text_target=__lowerCamelCase       ,							max_length=5       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							return_tensors="""pt"""							)

         assert batch.input_ids.shape == (2, 4_0_9_6)
         assert batch.attention_mask.shape == (2, 4_0_9_6)
         assert targets["input_ids"].shape == (2, 5)
         assert len(__lowerCamelCase							) == 2  # input_ids, attention_mask.





      def 							_lowercase					(		self    :       Optional[Any]							)     -> Tuple:






         """simple docstring"""







         UpperCAmelCase      					=  (
             """This is an example string that is used to test the original TF implementation against the HF"""
             """ implementation"""
         )

         UpperCAmelCase      					=  self._large_tokenizer(__lowerCamelCase							).input_ids

         self.assertListEqual(
             __lowerCamelCase       ,							[1_8_2, 1_1_7, 1_4_2, 5_8_7, 4_2_1_1, 1_2_0, 1_1_7, 2_6_3, 1_1_2, 8_0_4, 1_0_9, 8_5_6, 2_5_0_1_6, 3_1_3_7, 4_6_4, 1_0_9, 2_6_9_5_5, 3_1_3_7, 1]       ,							)







import argparse
import os
import re

import packaging.version


__a	 =      """examples/"""
__a	 =      {
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
__a	 =      {
    """init""": """src/diffusers/__init__.py""",
    """setup""": """setup.py""",
}
__a	 =      """README.md"""
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[Any]:
   with open(lowerCAmelCase_	,    """r"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      UpperCAmelCase      					=  f.read()
   UpperCAmelCase					,	UpperCAmelCase      					=  REPLACE_PATTERNS[pattern]
   UpperCAmelCase      					=  replace.replace("""VERSION"""	,    lowerCAmelCase_ )
   UpperCAmelCase      					=  re_pattern.sub(lowerCAmelCase_	,    lowerCAmelCase_ )
   with open(lowerCAmelCase_	,    """w"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      f.write(lowerCAmelCase_ )
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->int:
   for folder, directories, fnames in os.walk(lowerCAmelCase_ ):
      # Removing some of the folders with non-actively maintained examples from the walk
      if "research_projects" in directories:
         directories.remove("""research_projects""" )
      if "legacy" in directories:
         directories.remove("""legacy""" )
      for fname in fnames:
         if fname.endswith(""".py""" ):
            update_version_in_file(os.path.join(lowerCAmelCase_	,    lowerCAmelCase_ )	,    lowerCAmelCase_	,    pattern="""examples""" )
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_=False )    ->Optional[int]:
   for pattern, fname in REPLACE_FILES.items():
      update_version_in_file(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )
   if not patch:
      update_version_in_examples(lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->List[str]:
   UpperCAmelCase      					=  """🤗 Transformers currently provides the following architectures"""
   UpperCAmelCase      					=  """1. Want to contribute a new model?"""
   with open(lowerCAmelCase_	,    """r"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      UpperCAmelCase      					=  f.readlines()

   # Find the start of the list.
   UpperCAmelCase      					=  0
   while not lines[start_index].startswith(_start_prompt ):
      start_index += 1
   start_index += 1

   UpperCAmelCase      					=  start_index
   # Update the lines in the model list.
   while not lines[index].startswith(_end_prompt ):
      if lines[index].startswith("""1.""" ):
         UpperCAmelCase      					=  lines[index].replace(
             """https://huggingface.co/docs/diffusers/main/model_doc"""	,    """https://huggingface.co/docs/diffusers/model_doc"""	,    )
      index += 1

   with open(lowerCAmelCase_	,    """w"""	,    encoding="""utf-8"""	,    newline="""\n""" ) as f:
      f.writelines(lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->List[str]:
   with open(REPLACE_FILES["""init"""]	,    """r""" ) as f:
      UpperCAmelCase      					=  f.read()
   UpperCAmelCase      					=  REPLACE_PATTERNS["""init"""][0].search(lowerCAmelCase_ ).groups()[0]
   return packaging.version.parse(lowerCAmelCase_ )
def 	_UpperCamelCase					(    lowerCAmelCase_=False )    ->int:
   UpperCAmelCase      					=  get_version()
   if patch and default_version.is_devrelease:
      raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!""" )
   if default_version.is_devrelease:
      UpperCAmelCase      					=  default_version.base_version
   elif patch:
      UpperCAmelCase      					=  F"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
   else:
      UpperCAmelCase      					=  F"""{default_version.major}.{default_version.minor + 1}.0"""

   # Now let's ask nicely if that's the right one.
   UpperCAmelCase      					=  input(F"""Which version are you releasing? [{default_version}]""" )
   if len(lowerCAmelCase_ ) == 0:
      UpperCAmelCase      					=  default_version

   print(F"""Updating version to {version}.""" )
   global_version_update(lowerCAmelCase_	,    patch=lowerCAmelCase_ )
def 	_UpperCamelCase					(    )    ->int:
   UpperCAmelCase      					=  get_version()
   UpperCAmelCase      					=  F"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
   UpperCAmelCase      					=  current_version.base_version

   # Check with the user we got that right.
   UpperCAmelCase      					=  input(F"""Which version are we developing now? [{dev_version}]""" )
   if len(lowerCAmelCase_ ) == 0:
      UpperCAmelCase      					=  dev_version

   print(F"""Updating version to {version}.""" )
   global_version_update(lowerCAmelCase_ )


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
 __a	 =      argparse.ArgumentParser()
 parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
 parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
 __a	 =      parser.parse_args()
 if not args.post_release:
  pre_release_work(patch=args.patch)
 elif args.patch:
  print("""Nothing to do after a patch :-)""")
 else:
  post_release_work()



