
'''simple docstring'''






import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
 import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_							: Optional[int] =							logging.get_logger(__name__)

UpperCAmelCase_							: List[Any] =							{'vocab_file': 'vocab.txt'}

UpperCAmelCase_							: Union[str, Any] =							{
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

UpperCAmelCase_							: List[Any] =							{
    'openbmb/cpm-ant-10b': 1024,
}







def 		A_							(						_lowerCAmelCase :     str				):



      """simple docstring"""




      _lowerCamelCase					: Tuple		    = collections.OrderedDict()
      with open(_lowerCAmelCase					,						"r"					,						encoding="utf-8"				) as reader:
            _lowerCamelCase					: List[Any]		    = reader.readlines()
      for index, token in enumerate(_lowerCAmelCase				):
            _lowerCamelCase					: Any		    = token.rstrip("\n"				)
            _lowerCamelCase					: Union[str, Any]		    = index
      return vocab

class 			UpperCAmelCase__    (							A			):



    def __init__(						self   :							List[str],__A   :							Union[str, Any],__A   :							List[Any]="<unk>",__A   :							List[str]=2_0_0  ):
          _lowerCamelCase					: List[Any]		    = vocab
          _lowerCamelCase					: Optional[Any]		    = unk_token
          _lowerCamelCase					: List[str]		    = max_input_chars_per_word



    def 	lowerCamelCase_	(						self   :							Optional[int],__A   :							Any  ):
          _lowerCamelCase					: List[str]		    = list(__A  )
          if len(__A  ) > self.max_input_chars_per_word:
                return [self.unk_token]

          _lowerCamelCase					: List[str]		    = 0
          _lowerCamelCase					: Tuple		    = []
          while start < len(__A  ):
                _lowerCamelCase					: List[str]		    = len(__A  )
                _lowerCamelCase					: Any		    = None
                while start < end:
                      _lowerCamelCase					: Union[str, Any]		    = "".join(chars[start:end]  )
                      if substr in self.vocab:
                            _lowerCamelCase					: Optional[int]		    = substr
                            break
                      end -= 1
                if cur_substr is None:
                      sub_tokens.append(self.unk_token  )
                      start += 1
                else:
                      sub_tokens.append(__A  )
                      _lowerCamelCase					: Dict		    = end

          return sub_tokens


class 			UpperCAmelCase__    (							A			):
    lowerCAmelCase_					     =		VOCAB_FILES_NAMES
    lowerCAmelCase_					     =		PRETRAINED_VOCAB_FILES_MAP
    lowerCAmelCase_					     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCAmelCase_					     =		['input_ids', 'attention_mask']
    lowerCAmelCase_					     =		False



    def __init__(						self   :							List[str],__A   :							Dict,__A   :							Union[str, Any]="<d>",__A   :							Union[str, Any]="</d>",__A   :							Optional[Any]="<s>",__A   :							Any="</s>",__A   :							Optional[Any]="<pad>",__A   :							Tuple="<unk>",__A   :							Any="</n>",__A   :							Tuple="</_>",__A   :							Tuple="left",**__A   :							List[str],):
          requires_backends(self,["jieba"]  )
          super().__init__(
              bod_token=__A,eod_token=__A,bos_token=__A,eos_token=__A,pad_token=__A,unk_token=__A,line_token=__A,space_token=__A,padding_side=__A,**__A,)
          _lowerCamelCase					: int		    = bod_token
          _lowerCamelCase					: Any		    = eod_token
          _lowerCamelCase					: Optional[Any]		    = load_vocab(__A  )
          _lowerCamelCase					: Union[str, Any]		    = self.encoder[space_token]
          _lowerCamelCase					: Optional[Any]		    = self.encoder[line_token]

          del self.encoder[space_token]
          del self.encoder[line_token]

          _lowerCamelCase					: str		    = collections.OrderedDict(sorted(self.encoder.items(),key=lambda __A  : x[1]  )  )
          _lowerCamelCase					: List[str]		    = {v: k for k, v in self.encoder.items()}

          _lowerCamelCase					: int		    = WordpieceTokenizer(vocab=self.encoder,unk_token=self.unk_token  )



    @property
    def 	lowerCamelCase_	(						self   :							Dict  ):
          return self.encoder[self.bod_token]



    @property
    def 	lowerCamelCase_	(						self   :							List[str]  ):
          return self.encoder[self.eod_token]



    @property
    def 	lowerCamelCase_	(						self   :							List[str]  ):
          return self.encoder["\n"]



    @property
    def 	lowerCamelCase_	(						self   :							Any  ):
          return len(self.encoder  )



    def 	lowerCamelCase_	(						self   :							Optional[Any]  ):
          return dict(self.encoder,**self.added_tokens_encoder  )



    def 	lowerCamelCase_	(						self   :							str,__A   :							int  ):
          _lowerCamelCase					: Tuple		    = []
          for x in jieba.cut(__A,cut_all=__A  ):
                output_tokens.extend(self.wordpiece_tokenizer.tokenize(__A  )  )
          return output_tokens



    def 	lowerCamelCase_	(						self   :							Optional[Any],__A   :							str,**__A   :							List[Any]  ):
          _lowerCamelCase					: List[str]		    = [i for i in token_ids if i >= 0]
          _lowerCamelCase					: str		    = [
              x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
          ]
          return super()._decode(__A,**__A  )



    def 	lowerCamelCase_	(						self   :							Tuple,__A   :							Optional[Any]  ):
          return token in self.encoder



    def 	lowerCamelCase_	(						self   :							Any,__A   :							List[str]  ):
          return "".join(__A  )



    def 	lowerCamelCase_	(						self   :							Tuple,__A   :							Optional[Any]  ):
          return self.encoder.get(__A,self.encoder.get(self.unk_token  )  )



    def 	lowerCamelCase_	(						self   :							Union[str, Any],__A   :							Union[str, Any]  ):
          return self.decoder.get(__A,self.unk_token  )



    def 	lowerCamelCase_	(						self   :							str,__A   :							str,__A   :							Optional[str] = None  ):
          if os.path.isdir(__A  ):
                _lowerCamelCase					: List[Any]		    = os.path.join(
                    __A,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
          else:
                _lowerCamelCase					: Optional[Any]		    = (filename_prefix + "-" if filename_prefix else "") + save_directory
          _lowerCamelCase					: Any		    = 0
          if " " in self.encoder:
                _lowerCamelCase					: Optional[Any]		    = self.encoder[" "]
                del self.encoder[" "]
          if "\n" in self.encoder:
                _lowerCamelCase					: List[Any]		    = self.encoder["\n"]
                del self.encoder["\n"]
          _lowerCamelCase					: str		    = collections.OrderedDict(sorted(self.encoder.items(),key=lambda __A  : x[1]  )  )
          with open(__A,"w",encoding="utf-8"  ) as writer:
                for token, token_index in self.encoder.items():
                      if index != token_index:
                            logger.warning(
                                f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                " Please check that the vocabulary is not corrupted!"  )
                            _lowerCamelCase					: Dict		    = token_index
                      writer.write(token + "\n"  )
                      index += 1
          return (vocab_file,)



    def 	lowerCamelCase_	(						self   :							List[Any],__A   :							List[int],__A   :							List[int] = None  ):
          if token_ids_a is None:
                return [self.bos_token_id] + token_ids_a
          return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



    def 	lowerCamelCase_	(						self   :							Optional[int],__A   :							List[int],__A   :							Optional[List[int]] = None,__A   :							bool = False  ):

          if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=__A,token_ids_a=__A,already_has_special_tokens=__A  )

          if token_ids_a is not None:
                return [1] + ([0] * len(__A  )) + [1] + ([0] * len(__A  ))
          return [1] + ([0] * len(__A  ))






'''simple docstring'''






import argparse
import os
import re

import packaging.version


__UpperCamelCase					    =	"examples/"
__UpperCamelCase					    =	{
    "examples": (re.compile(R"^check_min_version\(\"[^\"]+\"\)\s*$", re.MULTILINE), "check_min_version(\"VERSION\")\n"),
    "init": (re.compile(R"^__version__\s+=\s+\"([^\"]+)\"\s*$", re.MULTILINE), "__version__ = \"VERSION\"\n"),
    "setup": (re.compile(R"^(\s*)version\s*=\s*\"[^\"]+\",", re.MULTILINE), R"\1version=\"VERSION\","),
    "doc": (re.compile(R"^(\s*)release\s*=\s*\"[^\"]+\"$", re.MULTILINE), "release = \"VERSION\"\n"),
}
__UpperCamelCase					    =	{
    "init": "src/transformers/__init__.py",
    "setup": "setup.py",
}
__UpperCamelCase					    =	"README.md"







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase ) ->				Tuple:



			"""simple docstring"""


			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      Union[str, Any]							      =     f.read()
			__snake_case      ,     __snake_case					:      List[Any]							      =     REPLACE_PATTERNS[pattern]
			__snake_case					:      Optional[Any]							      =     replace.replace("""VERSION"""    ,	_lowerCamelCase )
			__snake_case					:      Optional[Any]							      =     re_pattern.sub(_lowerCamelCase    ,	_lowerCamelCase )
			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.write(_lowerCamelCase )







def 					_a						(       _lowerCamelCase ) ->				Union[str, Any]:



			"""simple docstring"""


			for folder, directories, fnames in os.walk(_lowerCamelCase ):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove("""research_projects""" )
						if "legacy" in directories:
									directories.remove("""legacy""" )
						for fname in fnames:
									if fname.endswith(""".py""" ):
												update_version_in_file(os.path.join(_lowerCamelCase    ,	_lowerCamelCase )    ,	_lowerCamelCase    ,	pattern="""examples""" )







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase=False ) ->				str:



			"""simple docstring"""


			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(_lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase )
			if not patch:
						update_version_in_examples(_lowerCamelCase )







def 					_a						(       ) ->				Optional[int]:



			"""simple docstring"""


			__snake_case					:      str							      =     """🤗 Transformers currently provides the following architectures"""
			__snake_case					:      List[Any]							      =     """1. Want to contribute a new model?"""
			with open(_lowerCamelCase    ,	"""r"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						__snake_case					:      List[str]							      =     f.readlines()

			# Find the start of the list.
			__snake_case					:      Optional[Any]							      =     0
			while not lines[start_index].startswith(_start_prompt ):
						start_index += 1
			start_index += 1

			__snake_case					:      int							      =     start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt ):
						if lines[index].startswith("""1.""" ):
									__snake_case					:      Optional[Any]							      =     lines[index].replace(
									    """https://huggingface.co/docs/transformers/main/model_doc"""    ,	"""https://huggingface.co/docs/transformers/model_doc"""    ,	)
						index += 1

			with open(_lowerCamelCase    ,	"""w"""    ,	encoding="""utf-8"""    ,	newline="""\n""" ) as f:
						f.writelines(_lowerCamelCase )







def 					_a						(       ) ->				Union[str, Any]:



			"""simple docstring"""


			with open(REPLACE_FILES["""init"""]    ,	"""r""" ) as f:
						__snake_case					:      List[Any]							      =     f.read()
			__snake_case					:      str							      =     REPLACE_PATTERNS["""init"""][0].search(_lowerCamelCase ).groups()[0]
			return packaging.version.parse(_lowerCamelCase )







def 					_a						(       _lowerCamelCase=False ) ->				int:



			"""simple docstring"""


			__snake_case					:      List[Any]							      =     get_version()
			if patch and default_version.is_devrelease:
						raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!""" )
			if default_version.is_devrelease:
						__snake_case					:      str							      =     default_version.base_version
			elif patch:
						__snake_case					:      Optional[int]							      =     F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
			else:
						__snake_case					:      Dict							      =     F'''{default_version.major}.{default_version.minor + 1}.0'''

			# Now let's ask nicely if that's the right one.
			__snake_case					:      Dict							      =     input(F'''Which version are you releasing? [{default_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Any							      =     default_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase    ,	patch=_lowerCamelCase )
			if not patch:
						print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
						clean_main_ref_in_model_list()







def 					_a						(       ) ->				Tuple:



			"""simple docstring"""


			__snake_case					:      Optional[Any]							      =     get_version()
			__snake_case					:      Tuple							      =     F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
			__snake_case					:      Union[str, Any]							      =     current_version.base_version

			# Check with the user we got that right.
			__snake_case					:      int							      =     input(F'''Which version are we developing now? [{dev_version}]''' )
			if len(_lowerCamelCase ) == 0:
						__snake_case					:      Optional[int]							      =     dev_version

			print(F'''Updating version to {version}.''' )
			global_version_update(_lowerCamelCase )
			print("""Cleaning main README, don't forget to run `make fix-copies`.""" )
			clean_main_ref_in_model_list()


if __name__ == "__main__":
	__UpperCamelCase					    =	argparse.ArgumentParser()
	parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
	parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
	__UpperCamelCase					    =	parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print("Nothing to do after a patch :-)")
	else:
		post_release_work()


