



import argparse
import os
import re

import packaging.version


SCREAMING_SNAKE_CASE_:Union[str, Any] 			=		"""examples/"""
SCREAMING_SNAKE_CASE_:Tuple 			=		{
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
SCREAMING_SNAKE_CASE_:List[str] 			=		{
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
SCREAMING_SNAKE_CASE_:str 			=		"""README.md"""




def       __UpperCamelCase      (    _lowerCAmelCase ,	_lowerCAmelCase ,	_lowerCAmelCase						)			->    Optional[int]:



    """simple docstring"""






    with open(__lowerCamelCase ,	"""r""" ,	encoding="""utf-8""" ,	newline="""\n"""						) as f:
        A					:			Any		=  f.read()
    A	,				A					:			List[Any]		=  REPLACE_PATTERNS[pattern]
    A					:			Optional[Any]		=  replace.replace("""VERSION""" ,	__lowerCamelCase						)
    A					:			str		=  re_pattern.sub(__lowerCamelCase ,	__lowerCamelCase						)
    with open(__lowerCamelCase ,	"""w""" ,	encoding="""utf-8""" ,	newline="""\n"""						) as f:
        f.write(__lowerCamelCase						)




def       __UpperCamelCase      (    _lowerCAmelCase						)			->    str:



    """simple docstring"""






    for folder, directories, fnames in os.walk(__lowerCamelCase						):
        # Removing some of the folders with non-actively maintained examples from the walk
        if "research_projects" in directories:
            directories.remove("""research_projects"""						)
        if "legacy" in directories:
            directories.remove("""legacy"""						)
        for fname in fnames:
            if fname.endswith(""".py"""						):
                update_version_in_file(os.path.join(__lowerCamelCase ,	__lowerCamelCase						) ,	__lowerCamelCase ,	pattern="""examples"""						)




def       __UpperCamelCase      (    _lowerCAmelCase ,	_lowerCAmelCase=False						)			->    int:



    """simple docstring"""






    for pattern, fname in REPLACE_FILES.items():
        update_version_in_file(__lowerCamelCase ,	__lowerCamelCase ,	__lowerCamelCase						)
    if not patch:
        update_version_in_examples(__lowerCamelCase						)




def       __UpperCamelCase      (    )			->    Dict:



    """simple docstring"""






    A					:			int		=  """🤗 Transformers currently provides the following architectures"""
    A					:			Tuple		=  """1. Want to contribute a new model?"""
    with open(__lowerCamelCase ,	"""r""" ,	encoding="""utf-8""" ,	newline="""\n"""						) as f:
        A					:			Union[str, Any]		=  f.readlines()

    # Find the start of the list.
    A					:			int		=  0
    while not lines[start_index].startswith(_start_prompt						):
        start_index += 1
    start_index += 1

    A					:			int		=  start_index
    # Update the lines in the model list.
    while not lines[index].startswith(_end_prompt						):
        if lines[index].startswith("""1."""						):
            A					:			str		=  lines[index].replace(
                """https://huggingface.co/docs/transformers/main/model_doc""" ,	"""https://huggingface.co/docs/transformers/model_doc""" ,	)
        index += 1

    with open(__lowerCamelCase ,	"""w""" ,	encoding="""utf-8""" ,	newline="""\n"""						) as f:
        f.writelines(__lowerCamelCase						)




def       __UpperCamelCase      (    )			->    Union[str, Any]:



    """simple docstring"""






    with open(REPLACE_FILES["""init"""] ,	"""r"""						) as f:
        A					:			int		=  f.read()
    A					:			List[str]		=  REPLACE_PATTERNS["""init"""][0].search(__lowerCamelCase						).groups()[0]
    return packaging.version.parse(__lowerCamelCase						)




def       __UpperCamelCase      (    _lowerCAmelCase=False						)			->    Any:



    """simple docstring"""






    A					:			str		=  get_version()
    if patch and default_version.is_devrelease:
        raise ValueError("""Can't create a patch version from the dev branch, checkout a released version!"""						)
    if default_version.is_devrelease:
        A					:			Optional[int]		=  default_version.base_version
    elif patch:
        A					:			Union[str, Any]		=  f'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
    else:
        A					:			List[str]		=  f'''{default_version.major}.{default_version.minor + 1}.0'''

    # Now let's ask nicely if that's the right one.
    A					:			List[Any]		=  input(f'''Which version are you releasing? [{default_version}]'''						)
    if len(__lowerCamelCase						) == 0:
        A					:			Optional[Any]		=  default_version

    print(f'''Updating version to {version}.'''						)
    global_version_update(__lowerCamelCase ,	patch=__lowerCamelCase						)
    if not patch:
        print("""Cleaning main README, don't forget to run `make fix-copies`."""						)
        clean_main_ref_in_model_list()




def       __UpperCamelCase      (    )			->    Dict:



    """simple docstring"""






    A					:			Any		=  get_version()
    A					:			Optional[int]		=  f'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
    A					:			Dict		=  current_version.base_version

    # Check with the user we got that right.
    A					:			int		=  input(f'''Which version are we developing now? [{dev_version}]'''						)
    if len(__lowerCamelCase						) == 0:
        A					:			Union[str, Any]		=  dev_version

    print(f'''Updating version to {version}.'''						)
    global_version_update(__lowerCamelCase						)
    print("""Cleaning main README, don't forget to run `make fix-copies`."""						)
    clean_main_ref_in_model_list()


if __name__ == "__main__":
     SCREAMING_SNAKE_CASE_:str 			=		argparse.ArgumentParser()
     parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
     parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
     SCREAMING_SNAKE_CASE_:List[str] 			=		parser.parse_args()
     if not args.post_release:
          pre_release_work(patch=args.patch)
     elif args.patch:
          print("""Nothing to do after a patch :-)""")
     else:
          post_release_work()







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase_													=      logging.get_logger(__name__)

lowercase_													=      '''▁'''

lowercase_													=      {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

lowercase_													=      {
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

lowercase_													=      {
    '''facebook/m2m100_418M''': 10_24,
}

# fmt: off
lowercase_													=      {
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}






class     __a			(  SCREAMING_SNAKE_CASE					):
	SCREAMING_SNAKE_CASE  	=							VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE  	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE  	=							PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE  	=							["input_ids", "attention_mask"]

	SCREAMING_SNAKE_CASE  	=							[]
	SCREAMING_SNAKE_CASE  	=							[]



	def __init__(    self				:     int					,						snake_case_				:     List[str]					,						snake_case_				:     Any					,						snake_case_				:     Union[str, Any]=None					,						snake_case_				:     Tuple=None					,						snake_case_				:     List[str]="<s>"					,						snake_case_				:     List[str]="</s>"					,						snake_case_				:     Optional[Any]="</s>"					,						snake_case_				:     Dict="<pad>"					,						snake_case_				:     Any="<unk>"					,						snake_case_				:     Tuple="m2m100"					,						snake_case_				:     Optional[Dict[str, Any]] = None					,						snake_case_				:     Dict=8					,						**snake_case_				:     List[Any]					,						)->				None:
						__lowerCAmelCase						 ={} if sp_model_kwargs is None else sp_model_kwargs

						__lowerCAmelCase						 =language_codes
						__lowerCAmelCase						 =FAIRSEQ_LANGUAGE_CODES[language_codes]
						__lowerCAmelCase						 ={lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

						__lowerCAmelCase						 =kwargs.get("""additional_special_tokens"""					,						[])
						kwargs["additional_special_tokens"] += [
						    self.get_lang_token(snake_case_)
						    for lang_code in fairseq_language_code
						    if self.get_lang_token(snake_case_) not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=snake_case_					,						tgt_lang=snake_case_					,						bos_token=snake_case_					,						eos_token=snake_case_					,						sep_token=snake_case_					,						unk_token=snake_case_					,						pad_token=snake_case_					,						language_codes=snake_case_					,						sp_model_kwargs=self.sp_model_kwargs					,						num_madeup_words=snake_case_					,						**snake_case_					,						)

						__lowerCAmelCase						 =vocab_file
						__lowerCAmelCase						 =load_json(snake_case_)
						__lowerCAmelCase						 ={v: k for k, v in self.encoder.items()}
						__lowerCAmelCase						 =spm_file
						__lowerCAmelCase						 =load_spm(snake_case_					,						self.sp_model_kwargs)

						__lowerCAmelCase						 =len(self.encoder)

						__lowerCAmelCase						 ={
						    self.get_lang_token(snake_case_): self.encoder_size + i for i, lang_code in enumerate(snake_case_)
						}
						__lowerCAmelCase						 ={lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case_)}
						__lowerCAmelCase						 ={v: k for k, v in self.lang_token_to_id.items()}

						__lowerCAmelCase						 =src_lang if src_lang is not None else """en"""
						__lowerCAmelCase						 =tgt_lang
						__lowerCAmelCase						 =self.get_lang_id(self._src_lang)
						self.set_src_lang_special_tokens(self._src_lang)

						__lowerCAmelCase						 =num_madeup_words



	@property
	def 		UpperCamelCase				(    self				:     int)->				int:
						return len(self.encoder) + len(self.lang_token_to_id)



	@property
	def 		UpperCamelCase				(    self				:     Optional[Any])->				str:
						return self._src_lang



	@src_lang.setter
	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =new_src_lang
						self.set_src_lang_special_tokens(self._src_lang)



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     str)->				List[str]:
						return self.sp_model.encode(snake_case_					,						out_type=snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[Any]					,						snake_case_				:     List[str])->				str:
						if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
						return self.encoder.get(snake_case_					,						self.encoder[self.unk_token])



	def 		UpperCamelCase				(    self				:     Dict					,						snake_case_				:     int)->				str:
						if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
						return self.decoder.get(snake_case_					,						self.unk_token)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     Tuple)->				str:
						__lowerCAmelCase						 =[]
						__lowerCAmelCase						 =""""""
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																out_string += self.sp_model.decode(snake_case_) + token
																__lowerCAmelCase						 =[]
											else:
																current_sub_tokens.append(snake_case_)
						out_string += self.sp_model.decode(snake_case_)
						return out_string.strip()



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     List[int]					,						snake_case_				:     Optional[List[int]] = None					,						snake_case_				:     bool = False)->				List[int]:

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_					,						token_ids_a=snake_case_					,						already_has_special_tokens=snake_case_)

						__lowerCAmelCase						 =[1] * len(self.prefix_tokens)
						__lowerCAmelCase						 =[1] * len(self.suffix_tokens)
						if token_ids_a is None:
											return prefix_ones + ([0] * len(snake_case_)) + suffix_ones
						return prefix_ones + ([0] * len(snake_case_)) + ([0] * len(snake_case_)) + suffix_ones



	def 		UpperCamelCase				(    self				:     Any					,						snake_case_				:     List[int]					,						snake_case_				:     Optional[List[int]] = None)->				List[int]:
						if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Dict:
						__lowerCAmelCase						 ={self.convert_ids_to_tokens(snake_case_): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab



	def __getstate__(    self				:     Tuple)->				Dict:
						__lowerCAmelCase						 =self.__dict__.copy()
						__lowerCAmelCase						 =None
						return state



	def __setstate__(    self				:     List[Any]					,						snake_case_				:     Dict)->				None:
						__lowerCAmelCase						 =d

						# for backward compatibility
						if not hasattr(self					,						"""sp_model_kwargs"""):
											__lowerCAmelCase						 ={}

						__lowerCAmelCase						 =load_spm(self.spm_file					,						self.sp_model_kwargs)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     str					,						snake_case_				:     Optional[str] = None)->				Tuple[str]:
						__lowerCAmelCase						 =Path(snake_case_)
						if not save_dir.is_dir():
											raise OSError(F"""{save_directory} should be a directory""")
						__lowerCAmelCase						 =save_dir / (
						    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
						)
						__lowerCAmelCase						 =save_dir / (
						    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
						)

						save_json(self.encoder					,						snake_case_)

						if os.path.abspath(self.spm_file) != os.path.abspath(snake_case_) and os.path.isfile(self.spm_file):
											copyfile(self.spm_file					,						snake_case_)
						elif not os.path.isfile(self.spm_file):
											with open(snake_case_					,						"""wb""") as fi:
																__lowerCAmelCase						 =self.sp_model.serialized_model_proto()
																fi.write(snake_case_)

						return (str(snake_case_), str(snake_case_))



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     List[str]					,						snake_case_				:     str = "en"					,						snake_case_				:     Optional[List[str]] = None					,						snake_case_				:     str = "ro"					,						**snake_case_				:     int					,						)->				BatchEncoding:
						__lowerCAmelCase						 =src_lang
						__lowerCAmelCase						 =tgt_lang
						self.set_src_lang_special_tokens(self.src_lang)
						return super().prepare_seqaseq_batch(snake_case_					,						snake_case_					,						**snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     int					,						snake_case_				:     Optional[str]					,						snake_case_				:     Optional[str]					,						**snake_case_				:     List[str])->				List[Any]:
						if src_lang is None or tgt_lang is None:
											raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""")
						__lowerCAmelCase						 =src_lang
						__lowerCAmelCase						 =self(snake_case_					,						add_special_tokens=snake_case_					,						**snake_case_)
						__lowerCAmelCase						 =self.get_lang_id(snake_case_)
						__lowerCAmelCase						 =tgt_lang_id
						return inputs



	def 		UpperCamelCase				(    self				:     Optional[int])->				Union[str, Any]:
						self.set_src_lang_special_tokens(self.src_lang)



	def 		UpperCamelCase				(    self				:     Dict)->				Dict:
						self.set_tgt_lang_special_tokens(self.tgt_lang)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						__lowerCAmelCase						 =self.lang_token_to_id[lang_token]
						__lowerCAmelCase						 =[self.cur_lang_id]
						__lowerCAmelCase						 =[self.eos_token_id]



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						__lowerCAmelCase						 =self.lang_token_to_id[lang_token]
						__lowerCAmelCase						 =[self.cur_lang_id]
						__lowerCAmelCase						 =[self.eos_token_id]



	def 		UpperCamelCase				(    self				:     int					,						snake_case_				:     str)->				str:
						return self.lang_code_to_token[lang]



	def 		UpperCamelCase				(    self				:     List[str]					,						snake_case_				:     str)->				int:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						return self.lang_token_to_id[lang_token]






def        __lowerCAmelCase		(	__lowerCamelCase						:     str					, __lowerCamelCase						:     Dict[str, Any]						)	->    sentencepiece.SentencePieceProcessor:
					__lowerCAmelCase						 =sentencepiece.SentencePieceProcessor(**__lowerCamelCase						)
					spm.Load(str(__lowerCamelCase						)						)
					return spm






def        __lowerCAmelCase		(	__lowerCamelCase						:     str						)	->    Union[Dict, List]:
					with open(__lowerCamelCase					, """r"""						) as f:
										return json.load(__lowerCamelCase						)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Any					, __lowerCamelCase						:     str						)	->    None:
					with open(__lowerCamelCase					, """w"""						) as f:
										json.dump(__lowerCamelCase					, __lowerCamelCase					, indent=2						)




