






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			UpperCAmelCase__      			=						None


UpperCAmelCase__      			=						logging.get_logger(__name__)


UpperCAmelCase__      			=						{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__      			=						{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

UpperCAmelCase__      			=						{
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
UpperCAmelCase__      			=						["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]




class 							snake_case_       (    _SCREAMING_SNAKE_CASE      ):

						"""simple docstring"""



						snake_case__        =      VOCAB_FILES_NAMES
						snake_case__        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__        =      PRETRAINED_VOCAB_FILES_MAP
						snake_case__        =      ['''input_ids''', '''attention_mask''']
						snake_case__        =      MBartTokenizer

						snake_case__        =      []
						snake_case__        =      []



						def __init__(self:					str ,					__UpperCAmelCase:					Optional[Any]=None ,					__UpperCAmelCase:					str=None ,					__UpperCAmelCase:					Optional[Any]="<s>" ,					__UpperCAmelCase:					Any="</s>" ,					__UpperCAmelCase:					List[str]="</s>" ,					__UpperCAmelCase:					List[Any]="<s>" ,					__UpperCAmelCase:					List[Any]="<unk>" ,					__UpperCAmelCase:					str="<pad>" ,					__UpperCAmelCase:					Optional[int]="<mask>" ,					__UpperCAmelCase:					str=None ,					__UpperCAmelCase:					Tuple=None ,					__UpperCAmelCase:					Any=None ,					**__UpperCAmelCase:					int ,					)    ->     Optional[Any]:




											'''simple docstring'''



											__a					:						List[str]      =					AddedToken(__UpperCAmelCase ,					lstrip=__UpperCAmelCase ,					rstrip=__UpperCAmelCase   ) if isinstance(__UpperCAmelCase ,					__UpperCAmelCase   ) else mask_token

											super().__init__(
											    vocab_file=__UpperCAmelCase ,					tokenizer_file=__UpperCAmelCase ,					bos_token=__UpperCAmelCase ,					eos_token=__UpperCAmelCase ,					sep_token=__UpperCAmelCase ,					cls_token=__UpperCAmelCase ,					unk_token=__UpperCAmelCase ,					pad_token=__UpperCAmelCase ,					mask_token=__UpperCAmelCase ,					src_lang=__UpperCAmelCase ,					tgt_lang=__UpperCAmelCase ,					additional_special_tokens=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

											__a					:						Union[str, Any]      =					vocab_file
											__a					:						Optional[int]      =					False if not self.vocab_file else True

											__a					:						str      =					FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
																# Only add those special tokens if they are not already there.
																_additional_special_tokens.extend(
																    [t for t in additional_special_tokens if t not in _additional_special_tokens]   )

											self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}   )
											__a					:						Tuple      =					{
											    lang_code: self.convert_tokens_to_ids(__UpperCAmelCase   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											__a					:						Union[str, Any]      =					src_lang if src_lang is not None else '''en_XX'''
											__a					:						Optional[Any]      =					self.convert_tokens_to_ids(self._src_lang   )
											__a					:						Tuple      =					tgt_lang
											self.set_src_lang_special_tokens(self._src_lang   )



						@property
						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     str:




											'''simple docstring'''



											return self._src_lang



						@src_lang.setter
						def        UpperCAmelCase__					(self:					Optional[Any] ,					__UpperCAmelCase:					List[Any]   )    ->     None:




											'''simple docstring'''



											__a					:						List[Any]      =					new_src_lang
											self.set_src_lang_special_tokens(self._src_lang   )



						def        UpperCAmelCase__					(self:					Union[str, Any] ,					__UpperCAmelCase:					Union[str, Any] ,					__UpperCAmelCase:					Union[str, Any] = None   )    ->     List[int]:




											'''simple docstring'''



											if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					Union[str, Any] ,					__UpperCAmelCase:					List[str] = None   )    ->     List[int]:




											'''simple docstring'''



											__a					:						Dict      =					[self.sep_token_id]
											__a					:						str      =					[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					Dict ,					**__UpperCAmelCase:					Optional[Any]   )    ->     Optional[int]:




											'''simple docstring'''



											if src_lang is None or tgt_lang is None:
																raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"   )
											__a					:						Tuple      =					src_lang
											__a					:						Tuple      =					self(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						List[str]      =					self.convert_tokens_to_ids(__UpperCAmelCase   )
											__a					:						str      =					tgt_lang_id
											return inputs



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					Any = "en_XX" ,					__UpperCAmelCase:					int = None ,					__UpperCAmelCase:					Optional[Any] = "ro_RO" ,					**__UpperCAmelCase:					List[str] ,					)    ->     BatchEncoding:




											'''simple docstring'''



											__a					:						Optional[int]      =					src_lang
											__a					:						List[str]      =					tgt_lang
											return super().prepare_seqaseq_batch(__UpperCAmelCase ,					__UpperCAmelCase ,					**__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					List[str]   )    ->     List[str]:




											'''simple docstring'''



											return self.set_src_lang_special_tokens(self.src_lang   )



						def        UpperCAmelCase__					(self:					Optional[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											return self.set_tgt_lang_special_tokens(self.tgt_lang   )



						def        UpperCAmelCase__					(self:					Any ,					__UpperCAmelCase:					List[str]   )    ->     None:




											'''simple docstring'''



											__a					:						int      =					self.convert_tokens_to_ids(__UpperCAmelCase   )
											__a					:						Any      =					[]
											__a					:						Union[str, Any]      =					[self.eos_token_id, self.cur_lang_code]

											__a					:						List[str]      =					self.convert_ids_to_tokens(self.prefix_tokens   )
											__a					:						Optional[int]      =					self.convert_ids_to_tokens(self.suffix_tokens   )

											__a					:						Any      =					processors.TemplateProcessing(
											    single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,					pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,					self.prefix_tokens + self.suffix_tokens   )   ) ,					)



						def        UpperCAmelCase__					(self:					Optional[Any] ,					__UpperCAmelCase:					Union[str, Any]   )    ->     None:




											'''simple docstring'''



											__a					:						Any      =					self.convert_tokens_to_ids(__UpperCAmelCase   )
											__a					:						List[Any]      =					[]
											__a					:						Tuple      =					[self.eos_token_id, self.cur_lang_code]

											__a					:						Dict      =					self.convert_ids_to_tokens(self.prefix_tokens   )
											__a					:						Optional[Any]      =					self.convert_ids_to_tokens(self.suffix_tokens   )

											__a					:						Dict      =					processors.TemplateProcessing(
											    single=prefix_tokens_str + ["$A"] + suffix_tokens_str ,					pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str ,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,					self.prefix_tokens + self.suffix_tokens   )   ) ,					)



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Union[str, Any] ,					__UpperCAmelCase:					int = None   )    ->     Tuple[str]:




											'''simple docstring'''



											if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer."   )

											if not os.path.isdir(__UpperCAmelCase   ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory.'   )
																return
											__a					:						Union[str, Any]      =					os.path.join(
											    __UpperCAmelCase ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

											if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCAmelCase   ):
																copyfile(self.vocab_file ,					__UpperCAmelCase   )

											return (out_vocab_file,)








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase			:				Optional[int]   =     logging.get_logger(__name__)

lowercase			:				Optional[Any]   =     {"vocab_file": "vocab.txt"}

lowercase			:				Optional[int]   =     {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowercase			:				Tuple   =     {
    "YituTech/conv-bert-base": 5_12,
    "YituTech/conv-bert-medium-small": 5_12,
    "YituTech/conv-bert-small": 5_12,
}


lowercase			:				Optional[int]   =     {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class 					__lowercase  (						_SCREAMING_SNAKE_CASE				):





							"""simple docstring"""

							UpperCAmelCase_						:		int								=	VOCAB_FILES_NAMES
							UpperCAmelCase_						:		Optional[int]								=	PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_						:		List[Any]								=	PRETRAINED_INIT_CONFIGURATION
							UpperCAmelCase_						:		Optional[Any]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_						:		Optional[int]								=	ConvBertTokenizer




							def __init__(					self    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=None    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase="[UNK]"    ,						__UpperCAmelCase="[SEP]"    ,						__UpperCAmelCase="[PAD]"    ,						__UpperCAmelCase="[CLS]"    ,						__UpperCAmelCase="[MASK]"    ,						__UpperCAmelCase=True    ,						__UpperCAmelCase=None    ,						**__UpperCAmelCase    ,						)		->       Optional[int]:
														super().__init__(
														    __UpperCAmelCase    ,						tokenizer_file=__UpperCAmelCase    ,						do_lower_case=__UpperCAmelCase    ,						unk_token=__UpperCAmelCase    ,						sep_token=__UpperCAmelCase    ,						pad_token=__UpperCAmelCase    ,						cls_token=__UpperCAmelCase    ,						mask_token=__UpperCAmelCase    ,						tokenize_chinese_chars=__UpperCAmelCase    ,						strip_accents=__UpperCAmelCase    ,						**__UpperCAmelCase    ,						)

														A      :					Any      		=					json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
														if (
														    normalizer_state.get('''lowercase'''    ,						__UpperCAmelCase						) != do_lower_case
														    or normalizer_state.get('''strip_accents'''    ,						__UpperCAmelCase						) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''    ,						__UpperCAmelCase						) != tokenize_chinese_chars
														):
																					A      :					Tuple      		=					getattr(__UpperCAmelCase    ,						normalizer_state.pop('''type'''						)						)
																					A      :					int      		=					do_lower_case
																					A      :					int      		=					strip_accents
																					A      :					Union[str, Any]      		=					tokenize_chinese_chars
																					A      :					str      		=					normalizer_class(**__UpperCAmelCase						)

														A      :					str      		=					do_lower_case




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None						)		->       Optional[int]:
														A      :					Dict      		=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       List[int]:
														A      :					List[Any]      		=					[self.sep_token_id]
														A      :					Optional[Any]      		=					[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None						)		->       Tuple[str]:
														A      :					Optional[int]      		=					self._tokenizer.model.save(__UpperCAmelCase    ,						name=__UpperCAmelCase						)
														return tuple(__UpperCAmelCase						)






