





'''simple docstring'''






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _A (					__lowercase						,					unittest.TestCase		):
		lowercase__:   List[Any]				     =   None
		lowercase__:   Optional[int]				     =   BloomTokenizerFast
		lowercase__:   Union[str, Any]				     =   BloomTokenizerFast
		lowercase__:   int				     =   True
		lowercase__:   Tuple				     =   False
		lowercase__:   Union[str, Any]				     =   '''tokenizer_file'''
		lowercase__:   List[str]				     =   {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}

		def 	lowercase__     (   self					:   Optional[int]			)					->    Any:


					"""simple docstring"""

					super().setUp()
					__snake_case					:      List[Any]							      =     BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""			)
					tokenizer.save_pretrained(self.tmpdirname			)

		def 	lowercase__     (   self					:   Tuple  ,      **__magic_name__					:   List[Any]			)					->    List[Any]:


					"""simple docstring"""

					kwargs.update(self.special_tokens_map			)
					return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   Tuple			)					->    Any:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     self.get_rust_tokenizer()

					__snake_case					:      Optional[int]							      =     ["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
					__snake_case					:      Any							      =     [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

					__snake_case					:      Optional[int]							      =     tokenizer.batch_encode_plus(__magic_name__			)["""input_ids"""]
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

					__snake_case					:      List[Any]							      =     tokenizer.batch_decode(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   str=6			)					->    int:


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
											__snake_case					:      Optional[Any]							      =     self.rust_tokenizer_class.from_pretrained(__magic_name__  ,      **__magic_name__			)
											# tokenizer_r.pad_token = None # Hotfixing padding = None
											# Simple input
											__snake_case					:      List[Any]							      =     """This is a simple input"""
											__snake_case					:      List[Any]							      =     ["""This is a simple input 1""", """This is a simple input 2"""]
											__snake_case					:      int							      =     ("""This is a simple input""", """This is a pair""")
											__snake_case					:      List[Any]							      =     [
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											try:
														tokenizer_r.encode(__magic_name__  ,      max_length=__magic_name__			)
														tokenizer_r.encode_plus(__magic_name__  ,      max_length=__magic_name__			)

														tokenizer_r.batch_encode_plus(__magic_name__  ,      max_length=__magic_name__			)
														tokenizer_r.encode(__magic_name__  ,      max_length=__magic_name__			)
														tokenizer_r.batch_encode_plus(__magic_name__  ,      max_length=__magic_name__			)
											except ValueError:
														self.fail("""Bloom Tokenizer should be able to deal with padding"""			)

											__snake_case					:      Optional[int]							      =     None  # Hotfixing padding = None
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Simple input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(__magic_name__  ,      tokenizer_r.encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""			)

											# Pair input
											self.assertRaises(
											    __magic_name__  ,      tokenizer_r.batch_encode_plus  ,      __magic_name__  ,      max_length=__magic_name__  ,      padding="""max_length"""  ,      )

		def 	lowercase__     (   self					:   int			)					->    List[Any]:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     self.get_rust_tokenizer()
					__snake_case					:      List[Any]							      =     load_dataset("""xnli"""  ,      """all_languages"""  ,      split="""test"""  ,      streaming=__magic_name__			)

					__snake_case					:      str							      =     next(iter(__magic_name__			)			)["""premise"""]  # pick up one data
					__snake_case					:      Tuple							      =     list(sample_data.values()			)

					__snake_case					:      Optional[Any]							      =     list(map(tokenizer.encode  ,      __magic_name__			)			)
					__snake_case					:      Optional[int]							      =     [tokenizer.decode(__magic_name__  ,      clean_up_tokenization_spaces=__magic_name__			) for x in output_tokens]
					self.assertListEqual(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Optional[Any]			)					->    Optional[int]:


					"""simple docstring"""

					self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map			)  ,      1			)
					self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()			)[0]			)  ,      1			)









'''simple docstring'''






import argparse
import os
import re

import packaging.version


__UpperCamelCase					    =	"examples/"
__UpperCamelCase					    =	{
    "examples": (re.compile(R"^check_min_version\(\"[^\"]+\"\)\s*$", re.MULTILINE), "check_min_version(\"VERSION\")\n"),
    "init": (re.compile(R"^__version__\s+=\s+\"([^\"]+)\"\s*$", re.MULTILINE), "__version__ = \"VERSION\"\n"),
    "setup": (re.compile(R"^(\s*)version\s*=\s*\"[^\"]+\",", re.MULTILINE), R"\1version=\"VERSION\","),
    "doc": (re.compile(R"^(\s*)release\s*=\s*\"[^\"]+\"$", re.MULTILINE), "release = \"VERSION\"\n"),
}
__UpperCamelCase					    =	{
    "init": "src/transformers/__init__.py",
    "setup": "setup.py",
}
__UpperCamelCase					    =	"README.md"







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase ) ->				Tuple:



			"""simple docstring"""


			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      Union[str, Any]							      =     f.read()
			__snake_case      ,     __snake_case					:      List[Any]							      =     REPLACE_PATTERNS[pattern]
			__snake_case					:      Optional[Any]							      =     replace.replace("""VERSION"""    ,	_lowerCamelCase )
			__snake_case					:      Optional[Any]							      =     re_pattern.sub(_lowerCamelCase    ,	_lowerCamelCase )
			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.write(_lowerCamelCase )







def 					_a						(       _lowerCamelCase ) ->				Union[str, Any]:



			"""simple docstring"""


			for folder, directories, fnames in os.walk(_lowerCamelCase ):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove("""research_projects""" )
						if "legacy" in directories:
									directories.remove("""legacy""" )
						for fname in fnames:
									if fname.endswith(""".py""" ):
												update_version_in_file(os.path.join(_lowerCamelCase    ,	_lowerCamelCase )    ,	_lowerCamelCase    ,	pattern="""examples""" )







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase=False ) ->				str:



			"""simple docstring"""


			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(_lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase )
			if not patch:
						update_version_in_examples(_lowerCamelCase )







def 					_a						(       ) ->				Optional[int]:



			"""simple docstring"""


			__snake_case					:      str							      =     """🤗 Transformers currently provides the following architectures"""
			__snake_case					:      List[Any]							      =     """1. Want to contribute a new model?"""
			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      List[str]							      =     f.readlines()

			# Find the start of the list.
			__snake_case					:      Optional[Any]							      =     0
			while not lines[start_index].startswith(_start_prompt ):
						start_index += 1
			start_index += 1

			__snake_case					:      int							      =     start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt ):
						if lines[index].startswith("""1.""" ):
									__snake_case					:      Optional[Any]							      =     lines[index].replace(
									    """https://huggingface.co/docs/transformers/main/model_doc"""    ,	"""https://huggingface.co/docs/transformers/model_doc"""    ,	)
						index += 1

			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.writelines(_lowerCamelCase )







def 					_a						(       ) ->				Union[str, Any]:



			"""simple docstring"""


			with open(REPLACE_FILES["""init"""]    ,	"""r""" ) as f:
						__snake_case					:      List[Any]							      =     f.read()
			__snake_case					:      str							      =     REPLACE_PATTERNS["""init"""][0].search(_lowerCamelCase ).groups()[0]
			return packaging.version.parse(_lowerCamelCase )







def 					_a						(       _lowerCamelCase=False ) ->				int:



			"""simple docstring"""


			__snake_case					:      List[Any]							      =     get_version()
			if patch and default_version.is_devrelease:
						raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!""" )
			if default_version.is_devrelease:
						__snake_case					:      str							      =     default_version.base_version
			elif patch:
						__snake_case					:      Optional[int]							      =     F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
			else:
						__snake_case					:      Dict							      =     F'''{default_version.major}.{default_version.minor + 1}.0'''

			# Now let's ask nicely if that's the right one.
			__snake_case					:      Dict							      =     input(F'''Which version are you releasing? [{default_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Any							      =     default_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase    ,	patch=_lowerCamelCase )
			if not patch:
						print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
						clean_main_ref_in_model_list()







def 					_a						(       ) ->				Tuple:



			"""simple docstring"""


			__snake_case					:      Optional[Any]							      =     get_version()
			__snake_case					:      Tuple							      =     F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
			__snake_case					:      Union[str, Any]							      =     current_version.base_version

			# Check with the user we got that right.
			__snake_case					:      int							      =     input(F'''Which version are we developing now? [{dev_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Optional[int]							      =     dev_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase )
			print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
			clean_main_ref_in_model_list()


if __name__ == "__main__":
	__UpperCamelCase					    =	argparse.ArgumentParser()
	parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
	parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
	__UpperCamelCase					    =	parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print("Nothing to do after a patch :-)")
	else:
		post_release_work()


