





'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _A (					__lowercase						,					unittest.TestCase		):
		lowercase__:   Optional[int]				     =   CodeGenTokenizer
		lowercase__:   str				     =   CodeGenTokenizerFast
		lowercase__:   Dict				     =   True
		lowercase__:   List[str]				     =   {'''add_prefix_space''': True}
		lowercase__:   Optional[int]				     =   False

		def 	lowercase__     (   self					:   Dict			)					->    Any:


					"""simple docstring"""

					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					__snake_case					:      Optional[int]							      =     [
					    """l""",
					    """o""",
					    """w""",
					    """e""",
					    """r""",
					    """s""",
					    """t""",
					    """i""",
					    """d""",
					    """n""",
					    """\u0120""",
					    """\u0120l""",
					    """\u0120n""",
					    """\u0120lo""",
					    """\u0120low""",
					    """er""",
					    """\u0120lowest""",
					    """\u0120newer""",
					    """\u0120wider""",
					    """<unk>""",
					    """<|endoftext|>""",
					]
					__snake_case					:      Any							      =     dict(zip(__magic_name__  ,      range(len(__magic_name__			)			)			)			)
					__snake_case					:      Union[str, Any]							      =     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
					__snake_case					:      Union[str, Any]							      =     {"""unk_token""": """<unk>"""}

					__snake_case					:      Dict							      =     os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["""vocab_file"""]			)
					__snake_case					:      Tuple							      =     os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["""merges_file"""]			)
					with open(self.vocab_file  ,      """w"""  ,      encoding="""utf-8"""			) as fp:
								fp.write(json.dumps(__magic_name__			) + """\n"""			)
					with open(self.merges_file  ,      """w"""  ,      encoding="""utf-8"""			) as fp:
								fp.write("""\n""".join(__magic_name__			)			)

		def 	lowercase__     (   self					:   Dict  ,      **__magic_name__					:   Tuple			)					->    List[Any]:


					"""simple docstring"""

					kwargs.update(self.special_tokens_map			)
					return CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   Any  ,      **__magic_name__					:   Union[str, Any]			)					->    List[Any]:


					"""simple docstring"""

					kwargs.update(self.special_tokens_map			)
					return CodeGenTokenizerFast.from_pretrained(self.tmpdirname  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   int  ,      __magic_name__					:   Union[str, Any]			)					->    List[Any]:


					"""simple docstring"""

					__snake_case					:      int							      =     """lower newer"""
					__snake_case					:      Any							      =     """lower newer"""
					return input_text, output_text

		def 	lowercase__     (   self					:   List[str]			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     CodeGenTokenizer(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map			)
					__snake_case					:      List[Any]							      =     """lower newer"""
					__snake_case					:      int							      =     ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
					__snake_case					:      List[str]							      =     tokenizer.tokenize(__magic_name__  ,      add_prefix_space=__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					__snake_case					:      str							      =     tokens + [tokenizer.unk_token]
					__snake_case					:      Optional[int]							      =     [14, 15, 10, 9, 3, 2, 15, 19]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__			)  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					if not self.test_rust_tokenizer:
								return

					__snake_case					:      str							      =     self.get_tokenizer()
					__snake_case					:      int							      =     self.get_rust_tokenizer(add_prefix_space=__magic_name__			)

					__snake_case					:      List[Any]							      =     """lower newer"""

					# Testing tokenization
					__snake_case					:      int							      =     tokenizer.tokenize(__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Dict							      =     rust_tokenizer.tokenize(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing conversion to ids without special tokens
					__snake_case					:      str							      =     tokenizer.encode(__magic_name__  ,      add_special_tokens=__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Any							      =     rust_tokenizer.encode(__magic_name__  ,      add_special_tokens=__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing conversion to ids with special tokens
					__snake_case					:      Optional[int]							      =     self.get_rust_tokenizer(add_prefix_space=__magic_name__			)
					__snake_case					:      List[str]							      =     tokenizer.encode(__magic_name__  ,      add_prefix_space=__magic_name__			)
					__snake_case					:      Union[str, Any]							      =     rust_tokenizer.encode(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					# Testing the unknown token
					__snake_case					:      Dict							      =     tokens + [rust_tokenizer.unk_token]
					__snake_case					:      Any							      =     [14, 15, 10, 9, 3, 2, 15, 19]
					self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__magic_name__			)  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Any  ,      *__magic_name__					:   str  ,      **__magic_name__					:   int			)					->    int:


					"""simple docstring"""

					pass

		def 	lowercase__     (   self					:   Union[str, Any]  ,      __magic_name__					:   Tuple=15			)					->    Any:


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
											__snake_case					:      Any							      =     self.rust_tokenizer_class.from_pretrained(__magic_name__  ,      **__magic_name__			)

											# Simple input
											__snake_case					:      Any							      =     """This is a simple input"""
											__snake_case					:      int							      =     ["""This is a simple input 1""", """This is a simple input 2"""]
											__snake_case					:      int							      =     ("""This is a simple input""", """This is a pair""")
											__snake_case					:      Optional[int]							      =     [
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

		def 	lowercase__     (   self					:   Optional[Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      pad_token="""<pad>"""			)

					# Simple input
					__snake_case					:      Tuple							      =     """This is a simple input"""
					__snake_case					:      Union[str, Any]							      =     ["""This is a simple input looooooooong""", """This is a simple input"""]
					__snake_case					:      int							      =     ("""This is a simple input""", """This is a pair""")
					__snake_case					:      str							      =     [
					    ("""This is a simple input loooooong""", """This is a simple input"""),
					    ("""This is a simple pair loooooong""", """This is a simple pair"""),
					]

					__snake_case					:      Any							      =     tokenizer.pad_token_id

					__snake_case					:      List[str]							      =     tokenizer(__magic_name__  ,      padding="""max_length"""  ,      max_length=30  ,      return_tensors="""np"""			)
					__snake_case					:      List[Any]							      =     tokenizer(__magic_name__  ,      padding=__magic_name__  ,      truncate=__magic_name__  ,      return_tensors="""np"""			)
					__snake_case					:      Optional[Any]							      =     tokenizer(*__magic_name__  ,      padding="""max_length"""  ,      max_length=60  ,      return_tensors="""np"""			)
					__snake_case					:      Optional[Any]							      =     tokenizer(__magic_name__  ,      padding=__magic_name__  ,      truncate=__magic_name__  ,      return_tensors="""np"""			)

					# s
					# test single string max_length padding
					self.assertEqual(out_s["""input_ids"""].shape[-1]  ,      30			)
					self.assertTrue(pad_token_id in out_s["""input_ids"""]			)
					self.assertTrue(0 in out_s["""attention_mask"""]			)

					# s2
					# test automatic padding
					self.assertEqual(out_sa["""input_ids"""].shape[-1]  ,      33			)
					# long slice doesn't have padding
					self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]			)
					self.assertFalse(0 in out_sa["""attention_mask"""][0]			)
					# short slice does have padding
					self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]			)
					self.assertTrue(0 in out_sa["""attention_mask"""][1]			)

					# p
					# test single pair max_length padding
					self.assertEqual(out_p["""input_ids"""].shape[-1]  ,      60			)
					self.assertTrue(pad_token_id in out_p["""input_ids"""]			)
					self.assertTrue(0 in out_p["""attention_mask"""]			)

					# p2
					# test automatic padding pair
					self.assertEqual(out_pa["""input_ids"""].shape[-1]  ,      52			)
					# long slice pair doesn't have padding
					self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]			)
					self.assertFalse(0 in out_pa["""attention_mask"""][0]			)
					# short slice pair does have padding
					self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]			)
					self.assertTrue(0 in out_pa["""attention_mask"""][1]			)

		def 	lowercase__     (   self					:   Any			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      Optional[Any]							      =     """$$$"""
					__snake_case					:      Union[str, Any]							      =     CodeGenTokenizer.from_pretrained(self.tmpdirname  ,      bos_token=__magic_name__  ,      add_bos_token=__magic_name__			)

					__snake_case					:      Tuple							      =     """This is a simple input"""
					__snake_case					:      Dict							      =     ["""This is a simple input 1""", """This is a simple input 2"""]

					__snake_case					:      int							      =     tokenizer.bos_token_id

					__snake_case					:      List[Any]							      =     tokenizer(__magic_name__			)
					__snake_case					:      str							      =     tokenizer(__magic_name__			)

					self.assertEqual(out_s.input_ids[0]  ,      __magic_name__			)
					self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

					__snake_case					:      str							      =     tokenizer.decode(out_s.input_ids			)
					__snake_case					:      Tuple							      =     tokenizer.batch_decode(out_sa.input_ids			)

					self.assertEqual(decode_s.split()[0]  ,      __magic_name__			)
					self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)

		@slow
		def 	lowercase__     (   self					:   Union[str, Any]			)					->    int:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""			)

					__snake_case					:      Dict							      =     """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
					__snake_case					:      Any							      =     """\nif len_a > len_b:      result = a\nelse:      result = b"""

					__snake_case					:      Dict							      =     tokenizer.encode(__magic_name__			)
					__snake_case					:      Any							      =     ["""^#""", re.escape("""<|endoftext|>"""			), """^'''""", """^\"\"\"""", """\n\n\n"""]
					__snake_case					:      Optional[int]							      =     tokenizer.decode(__magic_name__  ,      truncate_before_pattern=__magic_name__			)
					self.assertEqual(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					pass









'''simple docstring'''






import argparse
import os
import re

import packaging.version


__UpperCamelCase					    =	"examples/"
__UpperCamelCase					    =	{
    "examples": (re.compile(R"^check_min_version\(\"[^\"]+\"\)\s*$", re.MULTILINE), "check_min_version(\"VERSION\")\n"),
    "init": (re.compile(R"^__version__\s+=\s+\"([^\"]+)\"\s*$", re.MULTILINE), "__version__ = \"VERSION\"\n"),
    "setup": (re.compile(R"^(\s*)version\s*=\s*\"[^\"]+\",", re.MULTILINE), R"\1version=\"VERSION\","),
    "doc": (re.compile(R"^(\s*)release\s*=\s*\"[^\"]+\"$", re.MULTILINE), "release = \"VERSION\"\n"),
}
__UpperCamelCase					    =	{
    "init": "src/transformers/__init__.py",
    "setup": "setup.py",
}
__UpperCamelCase					    =	"README.md"







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase ) ->				Tuple:



			"""simple docstring"""


			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      Union[str, Any]							      =     f.read()
			__snake_case      ,     __snake_case					:      List[Any]							      =     REPLACE_PATTERNS[pattern]
			__snake_case					:      Optional[Any]							      =     replace.replace("""VERSION"""    ,	_lowerCamelCase )
			__snake_case					:      Optional[Any]							      =     re_pattern.sub(_lowerCamelCase    ,	_lowerCamelCase )
			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.write(_lowerCamelCase )







def 					_a						(       _lowerCamelCase ) ->				Union[str, Any]:



			"""simple docstring"""


			for folder, directories, fnames in os.walk(_lowerCamelCase ):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove("""research_projects""" )
						if "legacy" in directories:
									directories.remove("""legacy""" )
						for fname in fnames:
									if fname.endswith(""".py""" ):
												update_version_in_file(os.path.join(_lowerCamelCase    ,	_lowerCamelCase )    ,	_lowerCamelCase    ,	pattern="""examples""" )







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase=False ) ->				str:



			"""simple docstring"""


			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(_lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase )
			if not patch:
						update_version_in_examples(_lowerCamelCase )







def 					_a						(       ) ->				Optional[int]:



			"""simple docstring"""


			__snake_case					:      str							      =     """🤗 Transformers currently provides the following architectures"""
			__snake_case					:      List[Any]							      =     """1. Want to contribute a new model?"""
			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      List[str]							      =     f.readlines()

			# Find the start of the list.
			__snake_case					:      Optional[Any]							      =     0
			while not lines[start_index].startswith(_start_prompt ):
						start_index += 1
			start_index += 1

			__snake_case					:      int							      =     start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt ):
						if lines[index].startswith("""1.""" ):
									__snake_case					:      Optional[Any]							      =     lines[index].replace(
									    """https://huggingface.co/docs/transformers/main/model_doc"""    ,	"""https://huggingface.co/docs/transformers/model_doc"""    ,	)
						index += 1

			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.writelines(_lowerCamelCase )







def 					_a						(       ) ->				Union[str, Any]:



			"""simple docstring"""


			with open(REPLACE_FILES["""init"""]    ,	"""r""" ) as f:
						__snake_case					:      List[Any]							      =     f.read()
			__snake_case					:      str							      =     REPLACE_PATTERNS["""init"""][0].search(_lowerCamelCase ).groups()[0]
			return packaging.version.parse(_lowerCamelCase )







def 					_a						(       _lowerCamelCase=False ) ->				int:



			"""simple docstring"""


			__snake_case					:      List[Any]							      =     get_version()
			if patch and default_version.is_devrelease:
						raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!""" )
			if default_version.is_devrelease:
						__snake_case					:      str							      =     default_version.base_version
			elif patch:
						__snake_case					:      Optional[int]							      =     F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
			else:
						__snake_case					:      Dict							      =     F'''{default_version.major}.{default_version.minor + 1}.0'''

			# Now let's ask nicely if that's the right one.
			__snake_case					:      Dict							      =     input(F'''Which version are you releasing? [{default_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Any							      =     default_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase    ,	patch=_lowerCamelCase )
			if not patch:
						print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
						clean_main_ref_in_model_list()







def 					_a						(       ) ->				Tuple:



			"""simple docstring"""


			__snake_case					:      Optional[Any]							      =     get_version()
			__snake_case					:      Tuple							      =     F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
			__snake_case					:      Union[str, Any]							      =     current_version.base_version

			# Check with the user we got that right.
			__snake_case					:      int							      =     input(F'''Which version are we developing now? [{dev_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Optional[int]							      =     dev_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase )
			print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
			clean_main_ref_in_model_list()


if __name__ == "__main__":
	__UpperCamelCase					    =	argparse.ArgumentParser()
	parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
	parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
	__UpperCamelCase					    =	parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print("Nothing to do after a patch :-)")
	else:
		post_release_work()


