



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__snake_case							:						Dict      =logging.get_logger(__name__)

__snake_case							:						Dict      ='▁'

__snake_case							:						Any      ={
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

__snake_case							:						Optional[int]      ={
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

__snake_case							:						str      ={
    'facebook/m2m100_418M': 1_0_2_4,
}

# fmt: off
__snake_case							:						Dict      ={
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}



class 			lowerCamelCase__     (				__A):




		'''simple docstring'''



		snake_case_      =VOCAB_FILES_NAMES
		snake_case_      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case_      =PRETRAINED_VOCAB_FILES_MAP
		snake_case_      =["""input_ids""", """attention_mask"""]

		snake_case_      =[]
		snake_case_      =[]



		def __init__(self						,__lowerCamelCase						,__lowerCamelCase						,__lowerCamelCase=None						,__lowerCamelCase=None						,__lowerCamelCase="<s>"						,__lowerCamelCase="</s>"						,__lowerCamelCase="</s>"						,__lowerCamelCase="<pad>"						,__lowerCamelCase="<unk>"						,__lowerCamelCase="m2m100"						,__lowerCamelCase = None						,__lowerCamelCase=8						,**__lowerCamelCase						,)	->     List[Any]:

						"""simple docstring"""






						lowerCAmelCase__	:							Any				=	{} if sp_model_kwargs is None else sp_model_kwargs

						lowerCAmelCase__	:							List[Any]				=	language_codes
						lowerCAmelCase__	:							Optional[Any]				=	FAIRSEQ_LANGUAGE_CODES[language_codes]
						lowerCAmelCase__	:							Dict				=	{lang_code: f"""__{lang_code}__""" for lang_code in fairseq_language_code}

						lowerCAmelCase__	:							Dict				=	kwargs.get('''additional_special_tokens'''						,[]					)
						kwargs["additional_special_tokens"] += [
						    self.get_lang_token(__A					)
						    for lang_code in fairseq_language_code
						    if self.get_lang_token(__A					) not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=__A						,tgt_lang=__A						,bos_token=__A						,eos_token=__A						,sep_token=__A						,unk_token=__A						,pad_token=__A						,language_codes=__A						,sp_model_kwargs=self.sp_model_kwargs						,num_madeup_words=__A						,**__A						,)

						lowerCAmelCase__	:							List[str]				=	vocab_file
						lowerCAmelCase__	:							Optional[int]				=	load_json(__A					)
						lowerCAmelCase__	:							int				=	{v: k for k, v in self.encoder.items()}
						lowerCAmelCase__	:							Dict				=	spm_file
						lowerCAmelCase__	:							List[str]				=	load_spm(__A						,self.sp_model_kwargs					)

						lowerCAmelCase__	:							Union[str, Any]				=	len(self.encoder					)

						lowerCAmelCase__	:							Dict				=	{
						    self.get_lang_token(__A					): self.encoder_size + i for i, lang_code in enumerate(__A					)
						}
						lowerCAmelCase__	:							Dict				=	{lang_code: self.encoder_size + i for i, lang_code in enumerate(__A					)}
						lowerCAmelCase__	:							Tuple				=	{v: k for k, v in self.lang_token_to_id.items()}

						lowerCAmelCase__	:							Union[str, Any]				=	src_lang if src_lang is not None else '''en'''
						lowerCAmelCase__	:							str				=	tgt_lang
						lowerCAmelCase__	:							Any				=	self.get_lang_id(self._src_lang					)
						self.set_src_lang_special_tokens(self._src_lang					)

						lowerCAmelCase__	:							Dict				=	num_madeup_words



		@property
		def 						lowerCAmelCase__ (self					)	->     Any:

						"""simple docstring"""






						return len(self.encoder					) + len(self.lang_token_to_id					)



		@property
		def 						lowerCAmelCase__ (self					)	->     Optional[Any]:

						"""simple docstring"""






						return self._src_lang



		@src_lang.setter
		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__	:							List[str]				=	new_src_lang
						self.set_src_lang_special_tokens(self._src_lang					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Optional[int]:

						"""simple docstring"""






						return self.sp_model.encode(__A						,out_type=__A					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Tuple:

						"""simple docstring"""






						if token in self.lang_token_to_id:
										return self.lang_token_to_id[token]
						return self.encoder.get(__A						,self.encoder[self.unk_token]					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Optional[Any]:

						"""simple docstring"""






						if index in self.id_to_lang_token:
										return self.id_to_lang_token[index]
						return self.decoder.get(__A						,self.unk_token					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Dict:

						"""simple docstring"""






						lowerCAmelCase__	:							Tuple				=	[]
						lowerCAmelCase__	:							Dict				=	''''''
						for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														out_string += self.sp_model.decode(__A					) + token
														lowerCAmelCase__	:							Tuple				=	[]
										else:
														current_sub_tokens.append(__A					)
						out_string += self.sp_model.decode(__A					)
						return out_string.strip()



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None						,__lowerCamelCase = False					)	->     Any:

						"""simple docstring"""






						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__A						,token_ids_a=__A						,already_has_special_tokens=__A					)

						lowerCAmelCase__	:							Optional[Any]				=	[1] * len(self.prefix_tokens					)
						lowerCAmelCase__	:							Tuple				=	[1] * len(self.suffix_tokens					)
						if token_ids_a is None:
										return prefix_ones + ([0] * len(__A					)) + suffix_ones
						return prefix_ones + ([0] * len(__A					)) + ([0] * len(__A					)) + suffix_ones



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None					)	->     Any:

						"""simple docstring"""






						if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def 						lowerCAmelCase__ (self					)	->     Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__	:							str				=	{self.convert_ids_to_tokens(__A					): i for i in range(self.vocab_size					)}
						vocab.update(self.added_tokens_encoder					)
						return vocab



		def __getstate__(self					)	->     Tuple:

						"""simple docstring"""






						lowerCAmelCase__	:							List[Any]				=	self.__dict__.copy()
						lowerCAmelCase__	:							int				=	None
						return state



		def __setstate__(self						,__lowerCamelCase					)	->     Any:

						"""simple docstring"""






						lowerCAmelCase__	:							List[Any]				=	d

						# for backward compatibility
						if not hasattr(self						,'''sp_model_kwargs'''					):
										lowerCAmelCase__	:							int				=	{}

						lowerCAmelCase__	:							List[str]				=	load_spm(self.spm_file						,self.sp_model_kwargs					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None					)	->     Any:

						"""simple docstring"""






						lowerCAmelCase__	:							Tuple				=	Path(__A					)
						if not save_dir.is_dir():
										raise OSError(f"""{save_directory} should be a directory"""					)
						lowerCAmelCase__	:							Optional[Any]				=	save_dir / (
						    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
						)
						lowerCAmelCase__	:							int				=	save_dir / (
						    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
						)

						save_json(self.encoder						,__A					)

						if os.path.abspath(self.spm_file					) != os.path.abspath(__A					) and os.path.isfile(self.spm_file					):
										copyfile(self.spm_file						,__A					)
						elif not os.path.isfile(self.spm_file					):
										with open(__A						,'''wb'''					) as fi:
														lowerCAmelCase__	:							Optional[Any]				=	self.sp_model.serialized_model_proto()
														fi.write(__A					)

						return (str(__A					), str(__A					))



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = "en"						,__lowerCamelCase = None						,__lowerCamelCase = "ro"						,**__lowerCamelCase						,)	->     Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__	:							int				=	src_lang
						lowerCAmelCase__	:							str				=	tgt_lang
						self.set_src_lang_special_tokens(self.src_lang					)
						return super().prepare_seqaseq_batch(__A						,__A						,**__A					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase						,__lowerCamelCase						,**__lowerCamelCase					)	->     int:

						"""simple docstring"""






						if src_lang is None or tgt_lang is None:
										raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''					)
						lowerCAmelCase__	:							Dict				=	src_lang
						lowerCAmelCase__	:							Optional[int]				=	self(__A						,add_special_tokens=__A						,**__A					)
						lowerCAmelCase__	:							str				=	self.get_lang_id(__A					)
						lowerCAmelCase__	:							str				=	tgt_lang_id
						return inputs



		def 						lowerCAmelCase__ (self					)	->     List[Any]:

						"""simple docstring"""






						self.set_src_lang_special_tokens(self.src_lang					)



		def 						lowerCAmelCase__ (self					)	->     int:

						"""simple docstring"""






						self.set_tgt_lang_special_tokens(self.tgt_lang					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Any:

						"""simple docstring"""






						lowerCAmelCase__	:							str				=	self.get_lang_token(__A					)
						lowerCAmelCase__	:							List[str]				=	self.lang_token_to_id[lang_token]
						lowerCAmelCase__	:							List[Any]				=	[self.cur_lang_id]
						lowerCAmelCase__	:							Dict				=	[self.eos_token_id]



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__	:							Optional[int]				=	self.get_lang_token(__A					)
						lowerCAmelCase__	:							Union[str, Any]				=	self.lang_token_to_id[lang_token]
						lowerCAmelCase__	:							Dict				=	[self.cur_lang_id]
						lowerCAmelCase__	:							int				=	[self.eos_token_id]



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Tuple:

						"""simple docstring"""






						return self.lang_code_to_token[lang]



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Tuple:

						"""simple docstring"""






						lowerCAmelCase__	:							List[str]				=	self.get_lang_token(__A					)
						return self.lang_token_to_id[lang_token]






def       lowerCAmelCase__						(       lowerCamelCase_						:						List[Any]			,lowerCamelCase_						:						Optional[int]):





				'''simple docstring'''

				lowerCAmelCase__	:							Any				=	sentencepiece.SentencePieceProcessor(**lowerCAmelCase__)
				spm.Load(str(lowerCAmelCase__))
				return spm






def       lowerCAmelCase__						(       lowerCamelCase_						:						Dict):





				'''simple docstring'''

				with open(lowerCAmelCase__			,'''r''') as f:
								return json.load(lowerCAmelCase__)






def       lowerCAmelCase__						(       lowerCamelCase_						:						int			,lowerCamelCase_						:						Union[str, Any]):





				'''simple docstring'''

				with open(lowerCAmelCase__			,'''w''') as f:
								json.dump(lowerCAmelCase__			,lowerCAmelCase__			,indent=2)





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							    =							{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE							    =							{
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

SCREAMING_SNAKE_CASE							    =							{'mobilebert-uncased': 5_1_2}


SCREAMING_SNAKE_CASE							    =							{}


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      VOCAB_FILES_NAMES
     _lowerCamelCase   =      PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase   =      PRETRAINED_INIT_CONFIGURATION
     _lowerCamelCase   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase   =      MobileBertTokenizer


     def __init__(	self  ,       __A=None  ,       __A=None  ,       __A=True  ,       __A="[UNK]"  ,       __A="[SEP]"  ,       __A="[PAD]"  ,       __A="[CLS]"  ,       __A="[MASK]"  ,       __A=True  ,       __A=None  ,       **__A  ,       ):
        super().__init__(
            __A  ,       tokenizer_file=__A  ,       do_lower_case=__A  ,       unk_token=__A  ,       sep_token=__A  ,       pad_token=__A  ,       cls_token=__A  ,       mask_token=__A  ,       tokenize_chinese_chars=__A  ,       strip_accents=__A  ,       **__A  ,       )

        __a					=  json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
        if (
            normalizer_state.get("""lowercase"""  ,       __A    ) != do_lower_case
            or normalizer_state.get("""strip_accents"""  ,       __A    ) != strip_accents
            or normalizer_state.get("""handle_chinese_chars"""  ,       __A    ) != tokenize_chinese_chars
        ):
           __a					=  getattr(__A  ,       normalizer_state.pop("""type"""    )    )
           __a					=  do_lower_case
           __a					=  strip_accents
           __a					=  tokenize_chinese_chars
           __a					=  normalizer_class(**__A    )

        __a					=  do_lower_case


     def 	snake_case_  (	self  ,       __A  ,       __A=None    ):
        __a					=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

        return output


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  [self.sep_token_id]
        __a					=  [self.cls_token_id]
        if token_ids_a is None:
           return len(cls + token_ids_a + sep    ) * [0]
        return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  self._tokenizer.model.save(__A  ,       name=__A    )
        return tuple(__A    )



