


import argparse
import os
import re

import packaging.version


lowerCamelCase__							:						Union[str, Any]							     =   """examples/"""
lowerCamelCase__							:						Union[str, Any]							     =   {
    """examples""": (re.compile(r"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(r"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(r"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), r"""\1version=\"VERSION\","""),
    """doc""": (re.compile(r"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
lowerCamelCase__							:						Dict							     =   {
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
lowerCamelCase__							:						List[Any]							     =   """README.md"""







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)    ->				List[Any]:
  with open(__lowerCAmelCase							,   '''r'''							,   encoding='''utf-8'''							,   newline='''\n'''				) as f:
    snake_case__						= f.read()
  snake_case__				,				snake_case__						= REPLACE_PATTERNS[pattern]
  snake_case__						= replace.replace('''VERSION'''							,   __lowerCAmelCase				)
  snake_case__						= re_pattern.sub(__lowerCAmelCase							,   __lowerCAmelCase				)
  with open(__lowerCAmelCase							,   '''w'''							,   encoding='''utf-8'''							,   newline='''\n'''				) as f:
    f.write(__lowerCAmelCase				)







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase				)    ->				Any:
  for folder, directories, fnames in os.walk(__lowerCAmelCase				):
    # Removing some of the folders with non-actively maintained examples from the walk
    if "research_projects" in directories:
      directories.remove('''research_projects'''				)
    if "legacy" in directories:
      directories.remove('''legacy'''				)
    for fname in fnames:
      if fname.endswith('''.py'''				):
        update_version_in_file(os.path.join(__lowerCAmelCase							,   __lowerCAmelCase				)							,   __lowerCAmelCase							,   pattern='''examples'''				)







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase=False				)    ->				int:
  for pattern, fname in REPLACE_FILES.items():
    update_version_in_file(__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)
  if not patch:
    update_version_in_examples(__lowerCAmelCase				)







def SCREAMING_SNAKE_CASE	(			)    ->				Dict:
  snake_case__						= '''🤗 Transformers currently provides the following architectures'''
  snake_case__						= '''1. Want to contribute a new model?'''
  with open(__lowerCAmelCase							,   '''r'''							,   encoding='''utf-8'''							,   newline='''\n'''				) as f:
    snake_case__						= f.readlines()

  # Find the start of the list.
  snake_case__						= 0
  while not lines[start_index].startswith(_start_prompt				):
    start_index += 1
  start_index += 1

  snake_case__						= start_index
  # Update the lines in the model list.
  while not lines[index].startswith(_end_prompt				):
    if lines[index].startswith('''1.'''				):
      snake_case__						= lines[index].replace(
          '''https://huggingface.co/docs/transformers/main/model_doc'''							,   '''https://huggingface.co/docs/transformers/model_doc'''							,   )
    index += 1

  with open(__lowerCAmelCase							,   '''w'''							,   encoding='''utf-8'''							,   newline='''\n'''				) as f:
    f.writelines(__lowerCAmelCase				)







def SCREAMING_SNAKE_CASE	(			)    ->				Any:
  with open(REPLACE_FILES['''init''']							,   '''r'''				) as f:
    snake_case__						= f.read()
  snake_case__						= REPLACE_PATTERNS['''init'''][0].search(__lowerCAmelCase				).groups()[0]
  return packaging.version.parse(__lowerCAmelCase				)







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase=False				)    ->				List[Any]:
  snake_case__						= get_version()
  if patch and default_version.is_devrelease:
    raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''				)
  if default_version.is_devrelease:
    snake_case__						= default_version.base_version
  elif patch:
    snake_case__						= F"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
  else:
    snake_case__						= F"""{default_version.major}.{default_version.minor + 1}.0"""

  # Now let's ask nicely if that's the right one.
  snake_case__						= input(F"""Which version are you releasing? [{default_version}]"""				)
  if len(__lowerCAmelCase				) == 0:
    snake_case__						= default_version

  print(F"""Updating version to {version}."""				)
  global_version_update(__lowerCAmelCase							,   patch=__lowerCAmelCase				)
  if not patch:
    print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''				)
    clean_main_ref_in_model_list()







def SCREAMING_SNAKE_CASE	(			)    ->				Any:
  snake_case__						= get_version()
  snake_case__						= F"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
  snake_case__						= current_version.base_version

  # Check with the user we got that right.
  snake_case__						= input(F"""Which version are we developing now? [{dev_version}]"""				)
  if len(__lowerCAmelCase				) == 0:
    snake_case__						= dev_version

  print(F"""Updating version to {version}."""				)
  global_version_update(__lowerCAmelCase				)
  print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''				)
  clean_main_ref_in_model_list()


if __name__ == "__main__":
   lowerCamelCase__							:						Optional[int]							     =   argparse.ArgumentParser()
   parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
   parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
   lowerCamelCase__							:						Tuple							     =   parser.parse_args()
   if not args.post_release:
      pre_release_work(patch=args.patch)
   elif args.patch:
      print("""Nothing to do after a patch :-)""")
   else:
      post_release_work()






'''simple docstring'''

import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     :						int	=  get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: List[Any]									=      GPTSwaTokenizer
			a_	: int									=      False
			a_	: List[Any]									=      True
			a_	: Any									=      False






			def 						_lowercase  (     self						)							->							str:
					super().setUp()

					# We have a SentencePiece fixture for testing
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase					,   eos_token='<unk>'					,   bos_token='<unk>'					,   pad_token='<unk>'						)

					tokenizer.save_pretrained(self.tmpdirname						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     ='This is a test'
					snake_case__	     ='This is a test'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							str:
					snake_case__	     ='<s>'
					snake_case__	     =1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase						)					,   _UpperCAmelCase						)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =list(self.get_tokenizer().get_vocab().keys()						)

					self.assertEqual(vocab_keys[0]					,   '<unk>'						)
					self.assertEqual(vocab_keys[1]					,   '<s>'						)
					self.assertEqual(vocab_keys[-1]					,   'j'						)
					self.assertEqual(len(_UpperCAmelCase						)					,   2000						)






			def 						_lowercase  (     self						)							->							List[str]:
					self.assertEqual(self.get_tokenizer().vocab_size					,   2000						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)

					snake_case__	     =tokenizer.tokenize('This is a test'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁This', '▁is', '▁a', '▁t', 'est']						)

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [465, 287, 265, 631, 842]						)

					snake_case__	     =tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']					,   )
					# fmt: on

					snake_case__	     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					,   )

					snake_case__	     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']						)
					# fmt: on






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)
					snake_case__	     =['This is a test', 'I was born in 92000, and this is falsé.']
					snake_case__	     =[
					    [465, 287, 265, 631, 842],
					    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
					]

					# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
					for text, expected_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertListEqual(tokenizer.encode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)

					# Test that decode_fast returns the input text
					for text, token_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertEqual(tokenizer.decode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			@slow
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =[
					    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
					    'Hey there, how are you doing this fine day?',
					    'This is a text with a trailing spaces followed by a dot     .',
					    'Häj sväjs lillebrör! =)',
					    'Det är inget fel på Mr. Cool',
					]

					# fmt: off
					snake_case__	     ={'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
					# fmt: on
					self.tokenizer_integration_test_util(
					    expected_encoding=_UpperCAmelCase					,   model_name='AI-Sweden/gpt-sw3-126m'					,   sequences=_UpperCAmelCase					,   )




