


'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  '''▁'''

__snake_case							   =  {'''vocab_file''': '''sentencepiece.bpe.model'''}

__snake_case							   =  {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

__snake_case							   =  {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
__snake_case							   =  ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']
class      lowercase			(							UpperCamelCase__      ):

		"""simple docstring"""

		_a									=       VOCAB_FILES_NAMES
		_a									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a									=       PRETRAINED_VOCAB_FILES_MAP
		_a									=       ["""input_ids""", """attention_mask"""]

		_a									=       []
		_a									=       []




		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_=None					,     UpperCamelCase_=None					,     UpperCamelCase_="</s>"					,     UpperCamelCase_="</s>"					,     UpperCamelCase_="<s>"					,     UpperCamelCase_="<unk>"					,     UpperCamelCase_="<pad>"					,     UpperCamelCase_="<mask>"					,     UpperCamelCase_ = None					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Optional[int]							      =				AddedToken(UpperCamelCase_					,     lstrip=UpperCamelCase_					,     rstrip=UpperCamelCase_							) if isinstance(UpperCamelCase_					,     UpperCamelCase_							) else mask_token

							UpperCamelCase__     :Optional[Any]							      =				{} if sp_model_kwargs is None else sp_model_kwargs

							UpperCamelCase__     :Tuple							      =				kwargs.get('''additional_special_tokens'''					,     []							)
							kwargs["additional_special_tokens"] += [
							    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=UpperCamelCase_					,     tgt_lang=UpperCamelCase_					,     eos_token=UpperCamelCase_					,     unk_token=UpperCamelCase_					,     sep_token=UpperCamelCase_					,     cls_token=UpperCamelCase_					,     pad_token=UpperCamelCase_					,     mask_token=UpperCamelCase_					,     sp_model_kwargs=self.sp_model_kwargs					,     **UpperCamelCase_					,     )

							UpperCamelCase__     :Optional[int]							      =				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(str(UpperCamelCase_							)							)
							UpperCamelCase__     :int							      =				vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCamelCase__     :Union[str, Any]							      =				{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCamelCase__     :List[str]							      =				1

							UpperCamelCase__     :Optional[int]							      =				len(self.sp_model							)
							UpperCamelCase__     :Any							      =				{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_							)
							}
							UpperCamelCase__     :Union[str, Any]							      =				{v: k for k, v in self.lang_code_to_id.items()}
							UpperCamelCase__     :str							      =				len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id							)
							UpperCamelCase__     :Dict							      =				{v: k for k, v in self.fairseq_tokens_to_ids.items()}

							UpperCamelCase__     :str							      =				src_lang if src_lang is not None else '''en_XX'''
							UpperCamelCase__     :str							      =				self.lang_code_to_id[self._src_lang]
							UpperCamelCase__     :Optional[int]							      =				tgt_lang
							self.set_src_lang_special_tokens(self._src_lang							)




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset + 1  # Plus 1 for the mask token




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return self._src_lang




		@src_lang.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				new_src_lang
							self.set_src_lang_special_tokens(self._src_lang							)




		def __getstate__(	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				self.__dict__.copy()
							UpperCamelCase__     :Any							      =				None
							return state




		def __setstate__(	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				d

							# for backward compatibility
							if not hasattr(self					,     '''sp_model_kwargs'''							):
												UpperCamelCase__     :str							      =				{}

							UpperCamelCase__     :Any							      =				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(self.vocab_file							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				{self.convert_ids_to_tokens(UpperCamelCase_							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							return self.sp_model.encode(UpperCamelCase_					,     out_type=UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCamelCase__     :Union[str, Any]							      =				self.sp_model.PieceToId(UpperCamelCase_							)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				[]
							UpperCamelCase__     :Any							      =				''''''
							UpperCamelCase__     :Union[str, Any]							      =				False
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	if not prev_is_special:
																						out_string += " "
																	out_string += self.sp_model.decode(UpperCamelCase_							) + token
																	UpperCamelCase__     :Optional[int]							      =				True
																	UpperCamelCase__     :Any							      =				[]
												else:
																	current_sub_tokens.append(UpperCamelCase_							)
																	UpperCamelCase__     :Tuple							      =				False
							out_string += self.sp_model.decode(UpperCamelCase_							)
							return out_string.strip()




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None							):

							'''simple docstring'''


							if not os.path.isdir(UpperCamelCase_							):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
												return
							UpperCamelCase__     :List[Any]							      =				os.path.join(
							    UpperCamelCase_					,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase_							) and os.path.isfile(self.vocab_file							):
												copyfile(self.vocab_file					,     UpperCamelCase_							)
							elif not os.path.isfile(self.vocab_file							):
												with open(UpperCamelCase_					,     '''wb'''							) as fi:
																	UpperCamelCase__     :Union[str, Any]							      =				self.sp_model.serialized_model_proto()
																	fi.write(UpperCamelCase_							)

							return (out_vocab_file,)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None					,     UpperCamelCase_ = False							):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_					,     token_ids_a=UpperCamelCase_					,     already_has_special_tokens=UpperCamelCase_							)

							UpperCamelCase__     :Dict							      =				[1] * len(self.prefix_tokens							)
							UpperCamelCase__     :int							      =				[1] * len(self.suffix_tokens							)
							if token_ids_a is None:
												return prefix_ones + ([0] * len(UpperCamelCase_							)) + suffix_ones
							return prefix_ones + ([0] * len(UpperCamelCase_							)) + ([0] * len(UpperCamelCase_							)) + suffix_ones




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = None							):

							'''simple docstring'''


							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							if src_lang is None or tgt_lang is None:
												raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''							)
							UpperCamelCase__     :Optional[int]							      =				src_lang
							UpperCamelCase__     :str							      =				self(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_					,     return_tensors=UpperCamelCase_					,     **UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				self.convert_tokens_to_ids(UpperCamelCase_							)
							UpperCamelCase__     :List[str]							      =				tgt_lang_id
							return inputs




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = "en_XX"					,     UpperCamelCase_ = None					,     UpperCamelCase_ = "ro_RO"					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				src_lang
							UpperCamelCase__     :int							      =				tgt_lang
							return super().prepare_seqaseq_batch(UpperCamelCase_					,     UpperCamelCase_					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return self.set_src_lang_special_tokens(self.src_lang							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							return self.set_tgt_lang_special_tokens(self.tgt_lang							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.lang_code_to_id[src_lang]
							UpperCamelCase__     :Any							      =				[self.cur_lang_code_id]
							UpperCamelCase__     :Tuple							      =				[self.eos_token_id]




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.lang_code_to_id[tgt_lang]
							UpperCamelCase__     :Tuple							      =				[self.cur_lang_code_id]
							UpperCamelCase__     :Tuple							      =				[self.eos_token_id]



'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__snake_case							   =  {
    '''configuration_layoutlmv3''': [
        '''LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LayoutLMv3Config''',
        '''LayoutLMv3OnnxConfig''',
    ],
    '''processing_layoutlmv3''': ['''LayoutLMv3Processor'''],
    '''tokenization_layoutlmv3''': ['''LayoutLMv3Tokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  ['''LayoutLMv3TokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  [
		    '''LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LayoutLMv3ForQuestionAnswering''',
		    '''LayoutLMv3ForSequenceClassification''',
		    '''LayoutLMv3ForTokenClassification''',
		    '''LayoutLMv3Model''',
		    '''LayoutLMv3PreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  [
		    '''TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFLayoutLMv3ForQuestionAnswering''',
		    '''TFLayoutLMv3ForSequenceClassification''',
		    '''TFLayoutLMv3ForTokenClassification''',
		    '''TFLayoutLMv3Model''',
		    '''TFLayoutLMv3PreTrainedModel''',
		]

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case							   =  ['''LayoutLMv3FeatureExtractor''']
		__snake_case							   =  ['''LayoutLMv3ImageProcessor''']


if TYPE_CHECKING:
		from .configuration_layoutlmva import (
		    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    LayoutLMvaConfig,
		    LayoutLMvaOnnxConfig,
		)
		from .processing_layoutlmva import LayoutLMvaProcessor
		from .tokenization_layoutlmva import LayoutLMvaTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_layoutlmva import (
				    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LayoutLMvaForQuestionAnswering,
				    LayoutLMvaForSequenceClassification,
				    LayoutLMvaForTokenClassification,
				    LayoutLMvaModel,
				    LayoutLMvaPreTrainedModel,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_layoutlmva import (
				    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFLayoutLMvaForQuestionAnswering,
				    TFLayoutLMvaForSequenceClassification,
				    TFLayoutLMvaForTokenClassification,
				    TFLayoutLMvaModel,
				    TFLayoutLMvaPreTrainedModel,
				)

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
				from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
		import sys

		__snake_case							   =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)