

'''simple docstring'''





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_							=					logging.get_logger(__name__)

UpperCamelCase_							=					"""▁"""

UpperCamelCase_							=					{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

UpperCamelCase_							=					{
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

UpperCamelCase_							=					{
    """facebook/m2m100_418M""": 1024,
}

# fmt: off
UpperCamelCase_							=					{
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}





class 		__SCREAMING_SNAKE_CASE       ( lowercase__			):
				lowerCamelCase_							=						VOCAB_FILES_NAMES
				lowerCamelCase_							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase_							=						PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_							=						['input_ids', 'attention_mask']

				lowerCamelCase_							=						[]
				lowerCamelCase_							=						[]







				def __init__(					self    :		List[Any]		,				UpperCAmelCase__    :		str		,				UpperCAmelCase__    :		List[str]		,				UpperCAmelCase__    :		List[Any]=None		,				UpperCAmelCase__    :		List[Any]=None		,				UpperCAmelCase__    :		Tuple="<s>"		,				UpperCAmelCase__    :		Tuple="</s>"		,				UpperCAmelCase__    :		int="</s>"		,				UpperCAmelCase__    :		Union[str, Any]="<pad>"		,				UpperCAmelCase__    :		Tuple="<unk>"		,				UpperCAmelCase__    :		str="m2m100"		,				UpperCAmelCase__    :		Optional[Dict[str, Any]] = None		,				UpperCAmelCase__    :		Union[str, Any]=8		,				**UpperCAmelCase__    :		Optional[int]		,				):
										'''simple docstring'''
										lowercase   :       Dict								={} if sp_model_kwargs is None else sp_model_kwargs

										lowercase   :       Any								=language_codes
										lowercase   :       Optional[Any]								=FAIRSEQ_LANGUAGE_CODES[language_codes]
										lowercase   :       Tuple								={lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

										lowercase   :       int								=kwargs.get('''additional_special_tokens'''		,				[]      )
										kwargs["additional_special_tokens"] += [
										    self.get_lang_token(UpperCAmelCase__      )
										    for lang_code in fairseq_language_code
										    if self.get_lang_token(UpperCAmelCase__      ) not in kwargs["additional_special_tokens"]
										]

										super().__init__(
										    src_lang=UpperCAmelCase__		,				tgt_lang=UpperCAmelCase__		,				bos_token=UpperCAmelCase__		,				eos_token=UpperCAmelCase__		,				sep_token=UpperCAmelCase__		,				unk_token=UpperCAmelCase__		,				pad_token=UpperCAmelCase__		,				language_codes=UpperCAmelCase__		,				sp_model_kwargs=self.sp_model_kwargs		,				num_madeup_words=UpperCAmelCase__		,				**UpperCAmelCase__		,				)

										lowercase   :       Union[str, Any]								=vocab_file
										lowercase   :       List[str]								=load_json(UpperCAmelCase__      )
										lowercase   :       List[str]								={v: k for k, v in self.encoder.items()}
										lowercase   :       Optional[Any]								=spm_file
										lowercase   :       Dict								=load_spm(UpperCAmelCase__		,				self.sp_model_kwargs      )

										lowercase   :       Optional[Any]								=len(self.encoder      )

										lowercase   :       Dict								={
										    self.get_lang_token(UpperCAmelCase__      ): self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase__      )
										}
										lowercase   :       Union[str, Any]								={lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCAmelCase__      )}
										lowercase   :       List[Any]								={v: k for k, v in self.lang_token_to_id.items()}

										lowercase   :       Optional[int]								=src_lang if src_lang is not None else '''en'''
										lowercase   :       Any								=tgt_lang
										lowercase   :       str								=self.get_lang_id(self._src_lang      )
										self.set_src_lang_special_tokens(self._src_lang      )

										lowercase   :       Optional[int]								=num_madeup_words







				@property
				def    lowerCamelCase_  (					self    :		Tuple      ):
										'''simple docstring'''
										return len(self.encoder      ) + len(self.lang_token_to_id      )







				@property
				def    lowerCamelCase_  (					self    :		Dict      ):
										'''simple docstring'''
										return self._src_lang







				@src_lang.setter
				def    lowerCamelCase_  (					self    :		List[Any]		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										lowercase   :       Tuple								=new_src_lang
										self.set_src_lang_special_tokens(self._src_lang      )







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										return self.sp_model.encode(UpperCAmelCase__		,				out_type=UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				UpperCAmelCase__    :		Optional[Any]      ):
										'''simple docstring'''
										if token in self.lang_token_to_id:
																return self.lang_token_to_id[token]
										return self.encoder.get(UpperCAmelCase__		,				self.encoder[self.unk_token]      )







				def    lowerCamelCase_  (					self    :		Optional[Any]		,				UpperCAmelCase__    :		int      ):
										'''simple docstring'''
										if index in self.id_to_lang_token:
																return self.id_to_lang_token[index]
										return self.decoder.get(UpperCAmelCase__		,				self.unk_token      )







				def    lowerCamelCase_  (					self    :		Optional[Any]		,				UpperCAmelCase__    :		Optional[Any]      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=[]
										lowercase   :       Union[str, Any]								=''''''
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						out_string += self.sp_model.decode(UpperCAmelCase__      ) + token
																						lowercase   :       Any								=[]
																else:
																						current_sub_tokens.append(UpperCAmelCase__      )
										out_string += self.sp_model.decode(UpperCAmelCase__      )
										return out_string.strip()







				def    lowerCamelCase_  (					self    :		Optional[int]		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None		,				UpperCAmelCase__    :		bool = False      ):
										'''simple docstring'''

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=UpperCAmelCase__		,				token_ids_a=UpperCAmelCase__		,				already_has_special_tokens=UpperCAmelCase__      )

										lowercase   :       int								=[1] * len(self.prefix_tokens      )
										lowercase   :       List[Any]								=[1] * len(self.suffix_tokens      )
										if token_ids_a is None:
																return prefix_ones + ([0] * len(UpperCAmelCase__      )) + suffix_ones
										return prefix_ones + ([0] * len(UpperCAmelCase__      )) + ([0] * len(UpperCAmelCase__      )) + suffix_ones







				def    lowerCamelCase_  (					self    :		str		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None      ):
										'''simple docstring'''
										if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







				def    lowerCamelCase_  (					self    :		Optional[int]      ):
										'''simple docstring'''
										lowercase   :       str								={self.convert_ids_to_tokens(UpperCAmelCase__      ): i for i in range(self.vocab_size      )}
										vocab.update(self.added_tokens_encoder      )
										return vocab







				def __getstate__(					self    :		Optional[int]      ):
										'''simple docstring'''
										lowercase   :       List[str]								=self.__dict__.copy()
										lowercase   :       int								=None
										return state







				def __setstate__(					self    :		str		,				UpperCAmelCase__    :		Dict      ):
										'''simple docstring'''
										lowercase   :       Optional[int]								=d

										# for backward compatibility
										if not hasattr(self		,				'''sp_model_kwargs'''      ):
																lowercase   :       Dict								={}

										lowercase   :       str								=load_spm(self.spm_file		,				self.sp_model_kwargs      )







				def    lowerCamelCase_  (					self    :		Dict		,				UpperCAmelCase__    :		str		,				UpperCAmelCase__    :		Optional[str] = None      ):
										'''simple docstring'''
										lowercase   :       Tuple								=Path(UpperCAmelCase__      )
										if not save_dir.is_dir():
																raise OSError(F'''{save_directory} should be a directory'''      )
										lowercase   :       Any								=save_dir / (
										    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
										)
										lowercase   :       Any								=save_dir / (
										    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
										)

										save_json(self.encoder		,				UpperCAmelCase__      )

										if os.path.abspath(self.spm_file      ) != os.path.abspath(UpperCAmelCase__      ) and os.path.isfile(self.spm_file      ):
																copyfile(self.spm_file		,				UpperCAmelCase__      )
										elif not os.path.isfile(self.spm_file      ):
																with open(UpperCAmelCase__		,				'''wb'''      ) as fi:
																						lowercase   :       Optional[Any]								=self.sp_model.serialized_model_proto()
																						fi.write(UpperCAmelCase__      )

										return (str(UpperCAmelCase__      ), str(UpperCAmelCase__      ))







				def    lowerCamelCase_  (					self    :		Optional[Any]		,				UpperCAmelCase__    :		List[str]		,				UpperCAmelCase__    :		str = "en"		,				UpperCAmelCase__    :		Optional[List[str]] = None		,				UpperCAmelCase__    :		str = "ro"		,				**UpperCAmelCase__    :		Any		,				):
										'''simple docstring'''
										lowercase   :       List[str]								=src_lang
										lowercase   :       Tuple								=tgt_lang
										self.set_src_lang_special_tokens(self.src_lang      )
										return super().prepare_seqaseq_batch(UpperCAmelCase__		,				UpperCAmelCase__		,				**UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Optional[Any]		,				UpperCAmelCase__    :		int		,				UpperCAmelCase__    :		Optional[str]		,				UpperCAmelCase__    :		Optional[str]		,				**UpperCAmelCase__    :		Union[str, Any]      ):
										'''simple docstring'''
										if src_lang is None or tgt_lang is None:
																raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''      )
										lowercase   :       Tuple								=src_lang
										lowercase   :       Union[str, Any]								=self(UpperCAmelCase__		,				add_special_tokens=UpperCAmelCase__		,				**UpperCAmelCase__      )
										lowercase   :       int								=self.get_lang_id(UpperCAmelCase__      )
										lowercase   :       Optional[Any]								=tgt_lang_id
										return inputs







				def    lowerCamelCase_  (					self    :		Dict      ):
										'''simple docstring'''
										self.set_src_lang_special_tokens(self.src_lang      )







				def    lowerCamelCase_  (					self    :		Optional[Any]      ):
										'''simple docstring'''
										self.set_tgt_lang_special_tokens(self.tgt_lang      )







				def    lowerCamelCase_  (					self    :		Dict		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										lowercase   :       Dict								=self.get_lang_token(UpperCAmelCase__      )
										lowercase   :       Optional[int]								=self.lang_token_to_id[lang_token]
										lowercase   :       Dict								=[self.cur_lang_id]
										lowercase   :       Optional[Any]								=[self.eos_token_id]







				def    lowerCamelCase_  (					self    :		Optional[int]		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=self.get_lang_token(UpperCAmelCase__      )
										lowercase   :       Union[str, Any]								=self.lang_token_to_id[lang_token]
										lowercase   :       Any								=[self.cur_lang_id]
										lowercase   :       Any								=[self.eos_token_id]







				def    lowerCamelCase_  (					self    :		str		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										return self.lang_code_to_token[lang]







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										lowercase   :       int								=self.get_lang_token(UpperCAmelCase__      )
										return self.lang_token_to_id[lang_token]






def 							_lowerCAmelCase (      __magic_name__	:    str      ,   __magic_name__	:    Dict[str, Any]	)							->     sentencepiece.SentencePieceProcessor:
						lowercase   :       Dict								=sentencepiece.SentencePieceProcessor(**__magic_name__	)
						spm.Load(str(__magic_name__	)	)
						return spm






def 							_lowerCAmelCase (      __magic_name__	:    str	)							->     Union[Dict, List]:
						with open(__magic_name__      ,   '''r'''	) as f:
												return json.load(__magic_name__	)







def 							_lowerCAmelCase (      __magic_name__	:    Tuple      ,   __magic_name__	:    str	)							->     None:
						with open(__magic_name__      ,   '''w'''	) as f:
												json.dump(__magic_name__      ,   __magic_name__      ,   indent=2	)








'''simple docstring'''





# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


UpperCamelCase_							=					"""3"""

print("""Python version:""", sys.version)

print("""OS platform:""", platform.platform())
print("""OS architecture:""", platform.machine())

try:
							import torch

							print("""Torch version:""", torch.__version__)
							print("""Cuda available:""", torch.cuda.is_available())
							print("""Cuda version:""", torch.version.cuda)
							print("""CuDNN version:""", torch.backends.cudnn.version())
							print("""Number of GPUs available:""", torch.cuda.device_count())
except ImportError:
							print("""Torch version:""", None)

try:
							import transformers

							print("""transformers version:""", transformers.__version__)
except ImportError:
							print("""transformers version:""", None)





