

"""simple docstring"""




import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class      __lowercase			(    __lowerCamelCase	,			unittest.TestCase					):
  snake_case_				       =       MgpstrTokenizer
  snake_case_				       =       False
  snake_case_				       =       {}
  snake_case_				       =       False

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   super().setUp()

   # fmt: off
   UpperCAmelCase__					:  Union[str, Any]					   =					["""[GO]""", """[s]""", """0""", """1""", """2""", """3""", """4""", """5""", """6""", """7""", """8""", """9""", """a""", """b""", """c""", """d""", """e""", """f""", """g""", """h""", """i""", """j""", """k""", """l""", """m""", """n""", """o""", """p""", """q""", """r""", """s""", """t""", """u""", """v""", """w""", """x""", """y""", """z"""]
   # fmt: on
   UpperCAmelCase__					:  List[str]					   =					dict(zip(A					,range(len(A    )    )    )    )

   UpperCAmelCase__					:  Any					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""vocab_file"""]    )
   with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""    ) as fp:
    fp.write(json.dumps(A    ) + """\n"""    )

  def    __lowercase       (  self		:			Tuple					,**A		:			Optional[Any]    ):






   '''simple docstring'''
   return MgpstrTokenizer.from_pretrained(self.tmpdirname					,**A    )

  def    __lowercase       (  self		:			str					,A		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					"""tester"""
   UpperCAmelCase__					:  Optional[Any]					   =					"""tester"""
   return input_text, output_text

  @unittest.skip("""MGP-STR always lower cases letters."""    )
  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   pass

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					self.get_tokenizers(do_lower_case=A    )
   for tokenizer in tokenizers:
    with self.subTest(f"{tokenizer.__class__.__name__}"    ):
     UpperCAmelCase__					:  List[str]					   =					"""[SPECIAL_TOKEN]"""

     tokenizer.add_special_tokens({"""cls_token""": special_token}    )
     UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.encode([special_token]					,add_special_tokens=A    )
     self.assertEqual(len(A    )					,1    )

     UpperCAmelCase__					:  int					   =					tokenizer.decode(A					,skip_special_tokens=A    )
     self.assertTrue(special_token not in decoded    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Dict					   =					self.get_tokenizers()
   for tokenizer in tokenizers:
    with self.subTest(f"{tokenizer.__class__.__name__}"    ):
     UpperCAmelCase__   ,      UpperCAmelCase__					:  int					   =					self.get_input_output_texts(A    )

     UpperCAmelCase__					:  Any					   =					tokenizer.tokenize(A    )
     UpperCAmelCase__					:  int					   =					tokenizer.convert_tokens_to_ids(A    )
     UpperCAmelCase__					:  Dict					   =					tokenizer.encode(A					,add_special_tokens=A    )
     self.assertListEqual(A					,A    )

     UpperCAmelCase__					:  int					   =					tokenizer.convert_ids_to_tokens(A    )
     self.assertNotEqual(len(A    )					,0    )
     UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.decode(A    )
     self.assertIsInstance(A					,A    )

     self.assertEqual(text_a.replace(""" """					,""""""    )					,A    )

  @unittest.skip("""MGP-STR tokenizer only handles one sequence."""    )
  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   pass




  @unittest.skip("""inputs cannot be pretokenized in MgpstrTokenizer"""    )
  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   pass








"""simple docstring"""




import argparse
import os
import re

import packaging.version


__UpperCAmelCase =				'examples/'
__UpperCAmelCase =				{
    'examples': (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'),
    'doc': (re.compile(r'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
__UpperCAmelCase =				{
    'init': 'src/transformers/__init__.py',
    'setup': 'setup.py',
}
__UpperCAmelCase =				'README.md'
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 with open(__UpperCamelCase							,    """r"""							,    encoding="""utf-8"""							,    newline="""\n"""      ) as f:
  UpperCAmelCase__					:  Dict					   =					f.read()
 UpperCAmelCase__   ,      UpperCAmelCase__					:  Any					   =					REPLACE_PATTERNS[pattern]
 UpperCAmelCase__					:  List[str]					   =					replace.replace("""VERSION"""							,    __UpperCamelCase      )
 UpperCAmelCase__					:  Tuple					   =					re_pattern.sub(__UpperCamelCase							,    __UpperCamelCase      )
 with open(__UpperCamelCase							,    """w"""							,    encoding="""utf-8"""							,    newline="""\n"""      ) as f:
  f.write(__UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 for folder, directories, fnames in os.walk(__UpperCamelCase      ):
  # Removing some of the folders with non-actively maintained examples from the walk
  if "research_projects" in directories:
   directories.remove("""research_projects"""      )
  if "legacy" in directories:
   directories.remove("""legacy"""      )
  for fname in fnames:
   if fname.endswith(""".py"""      ):
    update_version_in_file(os.path.join(__UpperCamelCase							,    __UpperCamelCase      )							,    __UpperCamelCase							,    pattern="""examples"""      )
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase=False      ):
 '''simple docstring'''





 for pattern, fname in REPLACE_FILES.items():
  update_version_in_file(__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      )
 if not patch:
  update_version_in_examples(__UpperCamelCase      )
def   lowerCAmelCase						(					):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					"""🤗 Transformers currently provides the following architectures"""
 UpperCAmelCase__					:  Union[str, Any]					   =					"""1. Want to contribute a new model?"""
 with open(__UpperCamelCase							,    """r"""							,    encoding="""utf-8"""							,    newline="""\n"""      ) as f:
  UpperCAmelCase__					:  str					   =					f.readlines()

 # Find the start of the list.
 UpperCAmelCase__					:  List[Any]					   =					0
 while not lines[start_index].startswith(_start_prompt      ):
  start_index += 1
 start_index += 1

 UpperCAmelCase__					:  int					   =					start_index
 # Update the lines in the model list.
 while not lines[index].startswith(_end_prompt      ):
  if lines[index].startswith("""1."""      ):
   UpperCAmelCase__					:  List[Any]					   =					lines[index].replace(
       """https://huggingface.co/docs/transformers/main/model_doc"""							,    """https://huggingface.co/docs/transformers/model_doc"""							,    )
  index += 1

 with open(__UpperCamelCase							,    """w"""							,    encoding="""utf-8"""							,    newline="""\n"""      ) as f:
  f.writelines(__UpperCamelCase      )
def   lowerCAmelCase						(					):
 '''simple docstring'''





 with open(REPLACE_FILES["""init"""]							,    """r"""      ) as f:
  UpperCAmelCase__					:  Optional[Any]					   =					f.read()
 UpperCAmelCase__					:  int					   =					REPLACE_PATTERNS["""init"""][0].search(__UpperCamelCase      ).groups()[0]
 return packaging.version.parse(__UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase=False      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[str]					   =					get_version()
 if patch and default_version.is_devrelease:
  raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!"""      )
 if default_version.is_devrelease:
  UpperCAmelCase__					:  List[str]					   =					default_version.base_version
 elif patch:
  UpperCAmelCase__					:  Any					   =					F"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
 else:
  UpperCAmelCase__					:  Optional[Any]					   =					F"{default_version.major}.{default_version.minor + 1}.0"

 # Now let's ask nicely if that's the right one.
 UpperCAmelCase__					:  Optional[int]					   =					input(F"Which version are you releasing? [{default_version}]"      )
 if len(__UpperCamelCase      ) == 0:
  UpperCAmelCase__					:  Union[str, Any]					   =					default_version

 print(F"Updating version to {version}."      )
 global_version_update(__UpperCamelCase							,    patch=__UpperCamelCase      )
 if not patch:
  print("""Cleaning main README, don't forget to run `make fix-copies`."""      )
  clean_main_ref_in_model_list()
def   lowerCAmelCase						(					):
 '''simple docstring'''





 UpperCAmelCase__					:  Optional[Any]					   =					get_version()
 UpperCAmelCase__					:  Any					   =					F"{current_version.major}.{current_version.minor + 1}.0.dev0"
 UpperCAmelCase__					:  List[Any]					   =					current_version.base_version

 # Check with the user we got that right.
 UpperCAmelCase__					:  Dict					   =					input(F"Which version are we developing now? [{dev_version}]"      )
 if len(__UpperCamelCase      ) == 0:
  UpperCAmelCase__					:  List[Any]					   =					dev_version

 print(F"Updating version to {version}."      )
 global_version_update(__UpperCamelCase      )
 print("""Cleaning main README, don't forget to run `make fix-copies`."""      )
 clean_main_ref_in_model_list()


if __name__ == "__main__":
      __UpperCAmelCase =				argparse.ArgumentParser()
      parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
      parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
      __UpperCAmelCase =				parser.parse_args()
      if not args.post_release:
            pre_release_work(patch=args.patch)
      elif args.patch:
            print('Nothing to do after a patch :-)')
      else:
            post_release_work()





