'''simple docstring'''




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowercase        =    logging.get_logger(__name__)

__lowercase        =    '''▁'''

__lowercase        =    {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__lowercase        =    {
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

__lowercase        =    {
    '''facebook/m2m100_418M''': 1_0_2_4,
}

# fmt: off
__lowercase        =    {
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}






class 							a__(   lowerCAmelCase__      ):


			'''simple docstring'''






			UpperCAmelCase_     :	Optional[Any]						=			VOCAB_FILES_NAMES
			UpperCAmelCase_     :	Any						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_     :	Any						=			PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_     :	int						=			['''input_ids''', '''attention_mask''']

			UpperCAmelCase_     :	List[int]						=			[]
			UpperCAmelCase_     :	List[int]						=			[]



			def __init__(  self			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       __lowerCAmelCase="<s>"			,       __lowerCAmelCase="</s>"			,       __lowerCAmelCase="</s>"			,       __lowerCAmelCase="<pad>"			,       __lowerCAmelCase="<unk>"			,       __lowerCAmelCase="m2m100"			,       __lowerCAmelCase = None			,       __lowerCAmelCase=8			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					lowerCAmelCase     =    {} if sp_model_kwargs is None else sp_model_kwargs

					lowerCAmelCase     =    language_codes
					lowerCAmelCase     =    FAIRSEQ_LANGUAGE_CODES[language_codes]
					lowerCAmelCase     =    {lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code}

					lowerCAmelCase     =    kwargs.get("""additional_special_tokens"""			,       [])
					kwargs["additional_special_tokens"] += [
					    self.get_lang_token(__lowerCAmelCase)
					    for lang_code in fairseq_language_code
					    if self.get_lang_token(__lowerCAmelCase) not in kwargs["additional_special_tokens"]
					]

					super().__init__(
					    src_lang=__lowerCAmelCase			,       tgt_lang=__lowerCAmelCase			,       bos_token=__lowerCAmelCase			,       eos_token=__lowerCAmelCase			,       sep_token=__lowerCAmelCase			,       unk_token=__lowerCAmelCase			,       pad_token=__lowerCAmelCase			,       language_codes=__lowerCAmelCase			,       sp_model_kwargs=self.sp_model_kwargs			,       num_madeup_words=__lowerCAmelCase			,       **__lowerCAmelCase			,       )

					lowerCAmelCase     =    vocab_file
					lowerCAmelCase     =    load_json(__lowerCAmelCase)
					lowerCAmelCase     =    {v: k for k, v in self.encoder.items()}
					lowerCAmelCase     =    spm_file
					lowerCAmelCase     =    load_spm(__lowerCAmelCase			,       self.sp_model_kwargs)

					lowerCAmelCase     =    len(self.encoder)

					lowerCAmelCase     =    {
					    self.get_lang_token(__lowerCAmelCase): self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase)
					}
					lowerCAmelCase     =    {lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase)}
					lowerCAmelCase     =    {v: k for k, v in self.lang_token_to_id.items()}

					lowerCAmelCase     =    src_lang if src_lang is not None else """en"""
					lowerCAmelCase     =    tgt_lang
					lowerCAmelCase     =    self.get_lang_id(self._src_lang)
					self.set_src_lang_special_tokens(self._src_lang)

					lowerCAmelCase     =    num_madeup_words



			@property
			def 				a_			(  self):




					"""simple docstring"""


					return len(self.encoder) + len(self.lang_token_to_id)



			@property
			def 				a_			(  self):




					"""simple docstring"""


					return self._src_lang



			@src_lang.setter
			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    new_src_lang
					self.set_src_lang_special_tokens(self._src_lang)



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					return self.sp_model.encode(__lowerCAmelCase			,       out_type=__lowerCAmelCase)



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					if token in self.lang_token_to_id:
							return self.lang_token_to_id[token]
					return self.encoder.get(__lowerCAmelCase			,       self.encoder[self.unk_token])



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					if index in self.id_to_lang_token:
							return self.id_to_lang_token[index]
					return self.decoder.get(__lowerCAmelCase			,       self.unk_token)



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    []
					lowerCAmelCase     =    """"""
					for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
									out_string += self.sp_model.decode(__lowerCAmelCase) + token
									lowerCAmelCase     =    []
							else:
									current_sub_tokens.append(__lowerCAmelCase)
					out_string += self.sp_model.decode(__lowerCAmelCase)
					return out_string.strip()



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None			,       __lowerCAmelCase = False):




					"""simple docstring"""


					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__lowerCAmelCase			,       token_ids_a=__lowerCAmelCase			,       already_has_special_tokens=__lowerCAmelCase)

					lowerCAmelCase     =    [1] * len(self.prefix_tokens)
					lowerCAmelCase     =    [1] * len(self.suffix_tokens)
					if token_ids_a is None:
							return prefix_ones + ([0] * len(__lowerCAmelCase)) + suffix_ones
					return prefix_ones + ([0] * len(__lowerCAmelCase)) + ([0] * len(__lowerCAmelCase)) + suffix_ones



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None):




					"""simple docstring"""


					if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    {self.convert_ids_to_tokens(__lowerCAmelCase): i for i in range(self.vocab_size)}
					vocab.update(self.added_tokens_encoder)
					return vocab



			def __getstate__(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.__dict__.copy()
					lowerCAmelCase     =    None
					return state



			def __setstate__(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    d

					# for backward compatibility
					if not hasattr(self			,       """sp_model_kwargs"""):
							lowerCAmelCase     =    {}

					lowerCAmelCase     =    load_spm(self.spm_file			,       self.sp_model_kwargs)



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase     =    Path(__lowerCAmelCase)
					if not save_dir.is_dir():
							raise OSError(f"{save_directory} should be a directory")
					lowerCAmelCase     =    save_dir / (
					    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
					)
					lowerCAmelCase     =    save_dir / (
					    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
					)

					save_json(self.encoder			,       __lowerCAmelCase)

					if os.path.abspath(self.spm_file) != os.path.abspath(__lowerCAmelCase) and os.path.isfile(self.spm_file):
							copyfile(self.spm_file			,       __lowerCAmelCase)
					elif not os.path.isfile(self.spm_file):
							with open(__lowerCAmelCase			,       """wb""") as fi:
									lowerCAmelCase     =    self.sp_model.serialized_model_proto()
									fi.write(__lowerCAmelCase)

					return (str(__lowerCAmelCase), str(__lowerCAmelCase))



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = "en"			,       __lowerCAmelCase = None			,       __lowerCAmelCase = "ro"			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					lowerCAmelCase     =    src_lang
					lowerCAmelCase     =    tgt_lang
					self.set_src_lang_special_tokens(self.src_lang)
					return super().prepare_seqaseq_batch(__lowerCAmelCase			,       __lowerCAmelCase			,       **__lowerCAmelCase)



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase			,       **__lowerCAmelCase):




					"""simple docstring"""


					if src_lang is None or tgt_lang is None:
							raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""")
					lowerCAmelCase     =    src_lang
					lowerCAmelCase     =    self(__lowerCAmelCase			,       add_special_tokens=__lowerCAmelCase			,       **__lowerCAmelCase)
					lowerCAmelCase     =    self.get_lang_id(__lowerCAmelCase)
					lowerCAmelCase     =    tgt_lang_id
					return inputs



			def 				a_			(  self):




					"""simple docstring"""


					self.set_src_lang_special_tokens(self.src_lang)



			def 				a_			(  self):




					"""simple docstring"""


					self.set_tgt_lang_special_tokens(self.tgt_lang)



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_lang_token(__lowerCAmelCase)
					lowerCAmelCase     =    self.lang_token_to_id[lang_token]
					lowerCAmelCase     =    [self.cur_lang_id]
					lowerCAmelCase     =    [self.eos_token_id]



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_lang_token(__lowerCAmelCase)
					lowerCAmelCase     =    self.lang_token_to_id[lang_token]
					lowerCAmelCase     =    [self.cur_lang_id]
					lowerCAmelCase     =    [self.eos_token_id]



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					return self.lang_code_to_token[lang]







			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_lang_token(__lowerCAmelCase)
					return self.lang_token_to_id[lang_token]







def 							snake_case__				(     _A:   str ,  _A:   Dict[str, Any]					)       ->      sentencepiece.SentencePieceProcessor:

		'''simple docstring'''


		lowerCAmelCase     =    sentencepiece.SentencePieceProcessor(**_A					)
		spm.Load(str(_A					)					)
		return spm






def 							snake_case__				(     _A:   str					)       ->      Union[Dict, List]:

		'''simple docstring'''


		with open(_A ,  """r"""					) as f:
				return json.load(_A					)






def 							snake_case__				(     _A:   str ,  _A:   str					)       ->      None:

		'''simple docstring'''


		with open(_A ,  """w"""					) as f:
				json.dump(_A ,  _A ,  indent=2					)





'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowercase        =    {
    '''configuration_chinese_clip''': [
        '''CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ChineseCLIPConfig''',
        '''ChineseCLIPOnnxConfig''',
        '''ChineseCLIPTextConfig''',
        '''ChineseCLIPVisionConfig''',
    ],
    '''processing_chinese_clip''': ['''ChineseCLIPProcessor'''],
}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase        =    ['''ChineseCLIPFeatureExtractor''']
						__lowercase        =    ['''ChineseCLIPImageProcessor''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase        =    [
						    '''CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''ChineseCLIPModel''',
						    '''ChineseCLIPPreTrainedModel''',
						    '''ChineseCLIPTextModel''',
						    '''ChineseCLIPVisionModel''',
						]

if TYPE_CHECKING:
						from .configuration_chinese_clip import (
						    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    ChineseCLIPConfig,
						    ChineseCLIPOnnxConfig,
						    ChineseCLIPTextConfig,
						    ChineseCLIPVisionConfig,
						)
						from .processing_chinese_clip import ChineseCLIPProcessor

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_chinese_clip import (
												    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
												    ChineseCLIPModel,
												    ChineseCLIPPreTrainedModel,
												    ChineseCLIPTextModel,
												    ChineseCLIPVisionModel,
												)

else:
						import sys

						__lowercase        =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




