





'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_mbart import MBartTokenizer
else:
	__UpperCamelCase					    =	None


__UpperCamelCase					    =	logging.get_logger(__name__)


__UpperCamelCase					    =	{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__UpperCamelCase					    =	{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

__UpperCamelCase					    =	{
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
__UpperCamelCase					    =	["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]




class  _A (					__lowercase		):
		lowercase__:   Dict				     =   VOCAB_FILES_NAMES
		lowercase__:   List[str]				     =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase__:   int				     =   PRETRAINED_VOCAB_FILES_MAP
		lowercase__:   Any				     =   ['''input_ids''', '''attention_mask''']
		lowercase__:   Union[str, Any]				     =   MBartTokenizer

		lowercase__:   Dict				     =   []
		lowercase__:   Tuple				     =   []

		def __init__(   self					:   Any  ,      __magic_name__					:   Optional[int]=None  ,      __magic_name__					:   int=None  ,      __magic_name__					:   Dict="<s>"  ,      __magic_name__					:   Tuple="</s>"  ,      __magic_name__					:   Union[str, Any]="</s>"  ,      __magic_name__					:   Optional[Any]="<s>"  ,      __magic_name__					:   Optional[Any]="<unk>"  ,      __magic_name__					:   Dict="<pad>"  ,      __magic_name__					:   Dict="<mask>"  ,      __magic_name__					:   Any=None  ,      __magic_name__					:   Optional[int]=None  ,      __magic_name__					:   Optional[int]=None  ,      **__magic_name__					:   str  ,      )					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      int							      =     AddedToken(__UpperCamelCase  ,      lstrip=__UpperCamelCase  ,      rstrip=__UpperCamelCase			) if isinstance(__UpperCamelCase  ,      __UpperCamelCase			) else mask_token

					super().__init__(
					    vocab_file=__UpperCamelCase  ,      tokenizer_file=__UpperCamelCase  ,      bos_token=__UpperCamelCase  ,      eos_token=__UpperCamelCase  ,      sep_token=__UpperCamelCase  ,      cls_token=__UpperCamelCase  ,      unk_token=__UpperCamelCase  ,      pad_token=__UpperCamelCase  ,      mask_token=__UpperCamelCase  ,      src_lang=__UpperCamelCase  ,      tgt_lang=__UpperCamelCase  ,      additional_special_tokens=__UpperCamelCase  ,      **__UpperCamelCase  ,      )

					__snake_case					:      int							      =     vocab_file
					__snake_case					:      int							      =     False if not self.vocab_file else True

					__snake_case					:      Optional[int]							      =     FAIRSEQ_LANGUAGE_CODES.copy()

					if additional_special_tokens is not None:
								# Only add those special tokens if they are not already there.
								_additional_special_tokens.extend(
								    [t for t in additional_special_tokens if t not in _additional_special_tokens]			)

					self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}			)
					__snake_case					:      Tuple							      =     {
					    lang_code: self.convert_tokens_to_ids(__UpperCamelCase			) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					__snake_case					:      Tuple							      =     src_lang if src_lang is not None else """en_XX"""
					__snake_case					:      Tuple							      =     self.convert_tokens_to_ids(self._src_lang			)
					__snake_case					:      int							      =     tgt_lang
					self.set_src_lang_special_tokens(self._src_lang			)

		@property
		def 	lowercase__     (   self					:   List[str]			)					->    Any:


					"""simple docstring"""

					return self._src_lang

		@src_lang.setter
		def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   str			)					->    int:


					"""simple docstring"""

					__snake_case					:      int							      =     new_src_lang
					self.set_src_lang_special_tokens(self._src_lang			)

		def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   List[int]  ,      __magic_name__					:   Optional[List[int]] = None			)					->    List[str]:


					"""simple docstring"""

					if token_ids_a is None:
								return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

		def 	lowercase__     (   self					:   Dict  ,      __magic_name__					:   List[int]  ,      __magic_name__					:   Optional[List[int]] = None			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      str							      =     [self.sep_token_id]
					__snake_case					:      Tuple							      =     [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep			) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]

		def 	lowercase__     (   self					:   int  ,      __magic_name__					:   Tuple  ,      __magic_name__					:   str  ,      __magic_name__					:   Optional[str]  ,      __magic_name__					:   Optional[str]  ,      **__magic_name__					:   Dict			)					->    Dict:


					"""simple docstring"""

					if src_lang is None or tgt_lang is None:
								raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""			)
					__snake_case					:      List[str]							      =     src_lang
					__snake_case					:      List[Any]							      =     self(__UpperCamelCase  ,      add_special_tokens=__UpperCamelCase  ,      return_tensors=__UpperCamelCase  ,      **__UpperCamelCase			)
					__snake_case					:      Tuple							      =     self.convert_tokens_to_ids(__UpperCamelCase			)
					__snake_case					:      Optional[int]							      =     tgt_lang_id
					return inputs

		def 	lowercase__     (   self					:   Tuple  ,      __magic_name__					:   List[str]  ,      __magic_name__					:   str = "en_XX"  ,      __magic_name__					:   Optional[List[str]] = None  ,      __magic_name__					:   str = "ro_RO"  ,      **__magic_name__					:   Optional[int]  ,      )					->    str:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     src_lang
					__snake_case					:      List[str]							      =     tgt_lang
					return super().prepare_seqaseq_batch(__UpperCamelCase  ,      __UpperCamelCase  ,      **__UpperCamelCase			)

		def 	lowercase__     (   self					:   int			)					->    int:


					"""simple docstring"""

					return self.set_src_lang_special_tokens(self.src_lang			)

		def 	lowercase__     (   self					:   Dict			)					->    Union[str, Any]:


					"""simple docstring"""

					return self.set_tgt_lang_special_tokens(self.tgt_lang			)

		def 	lowercase__     (   self					:   Dict  ,      __magic_name__					:   Dict			)					->    str:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     self.convert_tokens_to_ids(__UpperCamelCase			)
					__snake_case					:      Any							      =     []
					__snake_case					:      str							      =     [self.eos_token_id, self.cur_lang_code]

					__snake_case					:      str							      =     self.convert_ids_to_tokens(self.prefix_tokens			)
					__snake_case					:      List[str]							      =     self.convert_ids_to_tokens(self.suffix_tokens			)

					__snake_case					:      List[Any]							      =     processors.TemplateProcessing(
					    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str  ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str  ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,      self.prefix_tokens + self.suffix_tokens			)			)  ,      )

		def 	lowercase__     (   self					:   Dict  ,      __magic_name__					:   str			)					->    Tuple:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     self.convert_tokens_to_ids(__UpperCamelCase			)
					__snake_case					:      Any							      =     []
					__snake_case					:      List[Any]							      =     [self.eos_token_id, self.cur_lang_code]

					__snake_case					:      Optional[Any]							      =     self.convert_ids_to_tokens(self.prefix_tokens			)
					__snake_case					:      Tuple							      =     self.convert_ids_to_tokens(self.suffix_tokens			)

					__snake_case					:      str							      =     processors.TemplateProcessing(
					    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str  ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str  ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str  ,      self.prefix_tokens + self.suffix_tokens			)			)  ,      )

		def 	lowercase__     (   self					:   Tuple  ,      __magic_name__					:   str  ,      __magic_name__					:   Optional[str] = None			)					->    Tuple:


					"""simple docstring"""

					if not self.can_save_slow_tokenizer:
								raise ValueError(
								    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
								    """tokenizer."""			)

					if not os.path.isdir(__UpperCamelCase			):
								logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''			)
								return
					__snake_case					:      Union[str, Any]							      =     os.path.join(
					    __UpperCamelCase  ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

					if os.path.abspath(self.vocab_file			) != os.path.abspath(__UpperCamelCase			):
								copyfile(self.vocab_file  ,      __UpperCamelCase			)

					return (out_vocab_file,)




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


lowercase   =	{
    '''configuration_layoutlmv2''': ['''LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''LayoutLMv2Config'''],
    '''processing_layoutlmv2''': ['''LayoutLMv2Processor'''],
    '''tokenization_layoutlmv2''': ['''LayoutLMv2Tokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase   =	['''LayoutLMv2TokenizerFast''']

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase   =	['''LayoutLMv2FeatureExtractor''']
		lowercase   =	['''LayoutLMv2ImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase   =	[
		    '''LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LayoutLMv2ForQuestionAnswering''',
		    '''LayoutLMv2ForSequenceClassification''',
		    '''LayoutLMv2ForTokenClassification''',
		    '''LayoutLMv2Layer''',
		    '''LayoutLMv2Model''',
		    '''LayoutLMv2PreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_layoutlmva import LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMvaConfig
		from .processing_layoutlmva import LayoutLMvaProcessor
		from .tokenization_layoutlmva import LayoutLMvaTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor, LayoutLMvaImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_layoutlmva import (
				    LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LayoutLMvaForQuestionAnswering,
				    LayoutLMvaForSequenceClassification,
				    LayoutLMvaForTokenClassification,
				    LayoutLMvaLayer,
				    LayoutLMvaModel,
				    LayoutLMvaPreTrainedModel,
				)
else:
		import sys

		lowercase   =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





