


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


snake_case__			 =						logging.get_logger(__name__)

snake_case__			 =						'''▁'''

snake_case__			 =						{
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

snake_case__			 =						{
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

snake_case__			 =						{
    '''facebook/m2m100_418M''': 1024,
}

# fmt: off
snake_case__			 =						{
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}


class 					lowerCAmelCase_ (       A_):
						lowerCamelCase_         =     VOCAB_FILES_NAMES
						lowerCamelCase_         =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCamelCase_         =     PRETRAINED_VOCAB_FILES_MAP
						lowerCamelCase_         =     ['''input_ids''', '''attention_mask''']

						lowerCamelCase_         =     []
						lowerCamelCase_         =     []




						def __init__(	self						:   List[str]   ,     __A						:   Tuple   ,     __A						:   Tuple   ,     __A						:   List[Any]=None   ,     __A						:   Dict=None   ,     __A						:   List[str]="<s>"   ,     __A						:   Optional[int]="</s>"   ,     __A						:   Tuple="</s>"   ,     __A						:   List[Any]="<pad>"   ,     __A						:   Dict="<unk>"   ,     __A						:   Union[str, Any]="m2m100"   ,     __A						:   int = None   ,     __A						:   Optional[Any]=8   ,     **__A						:   List[str]   ,     )      ->List[str]:






									"""simple docstring"""






									a__ :str						  =    {} if sp_model_kwargs is None else sp_model_kwargs

									a__ :Any						  =    language_codes
									a__ :int						  =    FAIRSEQ_LANGUAGE_CODES[language_codes]
									a__ :Optional[Any]						  =    {lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

									a__ :Union[str, Any]						  =    kwargs.get("additional_special_tokens"   ,     []						)
									kwargs["additional_special_tokens"] += [
									    self.get_lang_token(__A						)
									    for lang_code in fairseq_language_code
									    if self.get_lang_token(__A						) not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    src_lang=__A   ,     tgt_lang=__A   ,     bos_token=__A   ,     eos_token=__A   ,     sep_token=__A   ,     unk_token=__A   ,     pad_token=__A   ,     language_codes=__A   ,     sp_model_kwargs=self.sp_model_kwargs   ,     num_madeup_words=__A   ,     **__A   ,     )

									a__ :Tuple						  =    vocab_file
									a__ :List[Any]						  =    load_json(__A						)
									a__ :List[Any]						  =    {v: k for k, v in self.encoder.items()}
									a__ :Union[str, Any]						  =    spm_file
									a__ :List[str]						  =    load_spm(__A   ,     self.sp_model_kwargs						)

									a__ :Tuple						  =    len(self.encoder						)

									a__ :List[str]						  =    {
									    self.get_lang_token(__A						): self.encoder_size + i for i, lang_code in enumerate(__A						)
									}
									a__ :List[Any]						  =    {lang_code: self.encoder_size + i for i, lang_code in enumerate(__A						)}
									a__ :List[Any]						  =    {v: k for k, v in self.lang_token_to_id.items()}

									a__ :int						  =    src_lang if src_lang is not None else """en"""
									a__ :int						  =    tgt_lang
									a__ :str						  =    self.get_lang_id(self._src_lang						)
									self.set_src_lang_special_tokens(self._src_lang						)

									a__ :List[str]						  =    num_madeup_words




						@property
						def  _snake_case					(	self						:   Tuple						)      ->Optional[int]:






									"""simple docstring"""






									return len(self.encoder						) + len(self.lang_token_to_id						)




						@property
						def  _snake_case					(	self						:   str						)      ->Union[str, Any]:






									"""simple docstring"""






									return self._src_lang




						@src_lang.setter
						def  _snake_case					(	self						:   Optional[int]   ,     __A						:   List[Any]						)      ->Any:






									"""simple docstring"""






									a__ :int						  =    new_src_lang
									self.set_src_lang_special_tokens(self._src_lang						)




						def  _snake_case					(	self						:   Any   ,     __A						:   int						)      ->List[Any]:






									"""simple docstring"""






									return self.sp_model.encode(__A   ,     out_type=__A						)




						def  _snake_case					(	self						:   List[str]   ,     __A						:   List[Any]						)      ->List[str]:






									"""simple docstring"""






									if token in self.lang_token_to_id:
												return self.lang_token_to_id[token]
									return self.encoder.get(__A   ,     self.encoder[self.unk_token]						)




						def  _snake_case					(	self						:   Union[str, Any]   ,     __A						:   List[str]						)      ->Optional[Any]:






									"""simple docstring"""






									if index in self.id_to_lang_token:
												return self.id_to_lang_token[index]
									return self.decoder.get(__A   ,     self.unk_token						)




						def  _snake_case					(	self						:   Dict   ,     __A						:   List[str]						)      ->Optional[int]:






									"""simple docstring"""






									a__ :List[Any]						  =    []
									a__ :List[str]						  =    """"""
									for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
															out_string += self.sp_model.decode(__A						) + token
															a__ :Union[str, Any]						  =    []
												else:
															current_sub_tokens.append(__A						)
									out_string += self.sp_model.decode(__A						)
									return out_string.strip()




						def  _snake_case					(	self						:   int   ,     __A						:   Dict   ,     __A						:   Optional[int] = None   ,     __A						:   Union[str, Any] = False						)      ->List[str]:






									"""simple docstring"""






									if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__A   ,     token_ids_a=__A   ,     already_has_special_tokens=__A						)

									a__ :int						  =    [1] * len(self.prefix_tokens						)
									a__ :Dict						  =    [1] * len(self.suffix_tokens						)
									if token_ids_a is None:
												return prefix_ones + ([0] * len(__A						)) + suffix_ones
									return prefix_ones + ([0] * len(__A						)) + ([0] * len(__A						)) + suffix_ones




						def  _snake_case					(	self						:   Any   ,     __A						:   str   ,     __A						:   Tuple = None						)      ->List[str]:






									"""simple docstring"""






									if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




						def  _snake_case					(	self						:   Optional[int]						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :Dict						  =    {self.convert_ids_to_tokens(__A						): i for i in range(self.vocab_size						)}
									vocab.update(self.added_tokens_encoder						)
									return vocab




						def __getstate__(	self						:   Any						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Any						  =    self.__dict__.copy()
									a__ :Any						  =    None
									return state




						def __setstate__(	self						:   int   ,     __A						:   int						)      ->Tuple:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    d

									# for backward compatibility
									if not hasattr(self   ,     "sp_model_kwargs"						):
												a__ :int						  =    {}

									a__ :List[str]						  =    load_spm(self.spm_file   ,     self.sp_model_kwargs						)




						def  _snake_case					(	self						:   Optional[Any]   ,     __A						:   Optional[Any]   ,     __A						:   List[Any] = None						)      ->List[str]:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    Path(__A						)
									if not save_dir.is_dir():
												raise OSError(F'''{save_directory} should be a directory'''						)
									a__ :Optional[Any]						  =    save_dir / (
									    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
									)
									a__ :Dict						  =    save_dir / (
									    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
									)

									save_json(self.encoder   ,     __A						)

									if os.path.abspath(self.spm_file						) != os.path.abspath(__A						) and os.path.isfile(self.spm_file						):
												copyfile(self.spm_file   ,     __A						)
									elif not os.path.isfile(self.spm_file						):
												with open(__A   ,     "wb"						) as fi:
															a__ :List[Any]						  =    self.sp_model.serialized_model_proto()
															fi.write(__A						)

									return (str(__A						), str(__A						))




						def  _snake_case					(	self						:   Optional[int]   ,     __A						:   Union[str, Any]   ,     __A						:   List[str] = "en"   ,     __A						:   str = None   ,     __A						:   int = "ro"   ,     **__A						:   List[Any]   ,     )      ->Tuple:






									"""simple docstring"""






									a__ :Any						  =    src_lang
									a__ :List[str]						  =    tgt_lang
									self.set_src_lang_special_tokens(self.src_lang						)
									return super().prepare_seqaseq_batch(__A   ,     __A   ,     **__A						)




						def  _snake_case					(	self						:   Tuple   ,     __A						:   List[Any]   ,     __A						:   List[str]   ,     __A						:   Any   ,     **__A						:   Optional[int]						)      ->Union[str, Any]:






									"""simple docstring"""






									if src_lang is None or tgt_lang is None:
												raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"						)
									a__ :List[str]						  =    src_lang
									a__ :Optional[Any]						  =    self(__A   ,     add_special_tokens=__A   ,     **__A						)
									a__ :List[Any]						  =    self.get_lang_id(__A						)
									a__ :Tuple						  =    tgt_lang_id
									return inputs




						def  _snake_case					(	self						:   str						)      ->int:






									"""simple docstring"""






									self.set_src_lang_special_tokens(self.src_lang						)




						def  _snake_case					(	self						:   Optional[Any]						)      ->Tuple:






									"""simple docstring"""






									self.set_tgt_lang_special_tokens(self.tgt_lang						)




						def  _snake_case					(	self						:   Any   ,     __A						:   List[Any]						)      ->Tuple:






									"""simple docstring"""






									a__ :int						  =    self.get_lang_token(__A						)
									a__ :str						  =    self.lang_token_to_id[lang_token]
									a__ :List[Any]						  =    [self.cur_lang_id]
									a__ :Tuple						  =    [self.eos_token_id]




						def  _snake_case					(	self						:   str   ,     __A						:   Optional[int]						)      ->Dict:






									"""simple docstring"""






									a__ :List[Any]						  =    self.get_lang_token(__A						)
									a__ :List[Any]						  =    self.lang_token_to_id[lang_token]
									a__ :Tuple						  =    [self.cur_lang_id]
									a__ :List[str]						  =    [self.eos_token_id]




						def  _snake_case					(	self						:   Dict   ,     __A						:   str						)      ->Tuple:






									"""simple docstring"""






									return self.lang_code_to_token[lang]






						def  _snake_case					(	self						:   Optional[int]   ,     __A						:   str						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Optional[int]						  =    self.get_lang_token(__A						)
									return self.lang_token_to_id[lang_token]






def 				lowerCamelCase__	(      a	:   str	,  a	:   Dict[str, Any]				)    ->	sentencepiece.SentencePieceProcessor:





			"""simple docstring"""
			a__ :List[Any]						  =    sentencepiece.SentencePieceProcessor(**a				)
			spm.Load(str(a				)				)
			return spm


def 				lowerCamelCase__	(      a	:   str				)    ->	Union[Dict, List]:





			"""simple docstring"""







			with open(a	,  "r"				) as f:
						return json.load(a				)







def 				lowerCamelCase__	(      a	:   Dict	,  a	:   str				)    ->	None:





			"""simple docstring"""







			with open(a	,  "w"				) as f:
						json.dump(a	,  a	,  indent=2				)






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


snake_case__			 =						[
    '''EAGER''',
    '''AOT_EAGER''',
    '''INDUCTOR''',
    '''NVFUSER''',
    '''AOT_NVFUSER''',
    '''AOT_CUDAGRAPHS''',
    '''OFI''',
    '''FX2TRT''',
    '''ONNXRT''',
    '''IPEX''',
]


def 				lowerCamelCase__	(      a	:   Optional[Any]	,  a	:   Tuple=None	,  a	:   int=None	,  a	:   List[str]=None				)    ->	int:





			"""simple docstring"""
			a__ :Tuple						  =    True
			while ask_again:
						a__ :str						  =    input(a				)
						try:
									if default is not None and len(a				) == 0:
												return default
									return convert_value(a				) if convert_value is not None else result
						except Exception:
									if error_message is not None:
												print(a				)


def 				lowerCamelCase__	(      a	:   int	,  a	:   Union[str, Any]=[]	,  a	:   str=None	,  a	:   str=0				)    ->	List[str]:





			"""simple docstring"""
			a__ :str						  =    BulletMenu(a	,  a				)
			a__ :int						  =    menu.run(default_choice=a				)
			return convert_value(a				) if convert_value is not None else result


def 				lowerCamelCase__	(      a	:   Optional[Any]				)    ->	Dict:





			"""simple docstring"""
			a__ :str						  =    int(a				)
			return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]				)


def 				lowerCamelCase__	(      a	:   Tuple				)    ->	List[str]:





			"""simple docstring"""
			a__ :Union[str, Any]						  =    int(a				)
			return DistributedType(["NO", "MULTI_CPU", "MULTI_XPU", "MULTI_GPU", "MULTI_NPU", "TPU"][value]				)


def 				lowerCamelCase__	(      a	:   List[Any]				)    ->	str:





			"""simple docstring"""
			a__ :int						  =    int(a				)
			return DynamoBackend(DYNAMO_BACKENDS[value]				).value


def 				lowerCamelCase__	(      a	:   Any				)    ->	Union[str, Any]:





			"""simple docstring"""
			a__ :Optional[Any]						  =    int(a				)
			return PrecisionType(["no", "fp16", "bf16", "fp8"][value]				)


def 				lowerCamelCase__	(      a	:   List[Any]				)    ->	Union[str, Any]:





			"""simple docstring"""
			a__ :Optional[Any]						  =    int(a				)
			return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]				)


def 				lowerCamelCase__	(      a	:   Union[str, Any]				)    ->	List[str]:





			"""simple docstring"""
			return {"yes": True, "no": False}[value.lower()]







class 					lowerCAmelCase_ (       argparse.RawDescriptionHelpFormatter):




						def  _snake_case					(	self						:   Tuple   ,     __A						:   Tuple   ,     __A						:   int   ,     __A						:   Tuple   ,     __A						:   Any						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Optional[int]						  =    super()._format_usage(__A   ,     __A   ,     __A   ,     __A						)
									a__ :Any						  =    usage.replace("<command> [<args>] "   ,     ""						)
									return usage


