


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase    		=       logging.get_logger(__name__)

__UpperCamelCase    		=       '▁'

__UpperCamelCase    		=       {'vocab_file': 'sentencepiece.bpe.model'}

__UpperCamelCase    		=       {
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__UpperCamelCase    		=       {
    'facebook/mbart-large-50-one-to-many-mmt': 1_0_2_4,
}

# fmt: off
__UpperCamelCase    		=       ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']







class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					Dict    =		VOCAB_FILES_NAMES
			_UpperCamelCase		:					Tuple    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase		:					Optional[Any]    =		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase		:					int    =		['input_ids', 'attention_mask']

			_UpperCamelCase		:					List[int]    =		[]
			_UpperCamelCase		:					List[int]    =		[]




			def __init__(							self      ,						snake_case      ,						snake_case=None      ,						snake_case=None      ,						snake_case="</s>"      ,						snake_case="</s>"      ,						snake_case="<s>"      ,						snake_case="<unk>"      ,						snake_case="<pad>"      ,						snake_case="<mask>"      ,						snake_case = None      ,						**snake_case      ,						):

						'''simple docstring'''


						UpperCamelCase__					=       AddedToken(snake_case      ,						lstrip=snake_case      ,						rstrip=snake_case     ) if isinstance(snake_case      ,						snake_case     ) else mask_token

						UpperCamelCase__					=       {} if sp_model_kwargs is None else sp_model_kwargs

						UpperCamelCase__					=       kwargs.get("additional_special_tokens"      ,						[]     )
						kwargs["additional_special_tokens"] += [
						    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=snake_case      ,						tgt_lang=snake_case      ,						eos_token=snake_case      ,						unk_token=snake_case      ,						sep_token=snake_case      ,						cls_token=snake_case      ,						pad_token=snake_case      ,						mask_token=snake_case      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**snake_case      ,						)

						UpperCamelCase__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(str(snake_case     )     )
						UpperCamelCase__					=       vocab_file

						# Original fairseq vocab and spm vocab must be "aligned":
						# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
						# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
						# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
						# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

						# Mimic fairseq token-to-id alignment for the first 4 token
						UpperCamelCase__					=       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

						# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
						UpperCamelCase__					=       1

						UpperCamelCase__					=       len(self.sp_model     )
						UpperCamelCase__					=       {
						    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(snake_case     )
						}
						UpperCamelCase__					=       {v: k for k, v in self.lang_code_to_id.items()}
						UpperCamelCase__					=       len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

						self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
						UpperCamelCase__					=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}

						UpperCamelCase__					=       src_lang if src_lang is not None else "en_XX"
						UpperCamelCase__					=       self.lang_code_to_id[self._src_lang]
						UpperCamelCase__					=       tgt_lang
						self.set_src_lang_special_tokens(self._src_lang     )




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return self._src_lang




			@src_lang.setter
			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       new_src_lang
						self.set_src_lang_special_tokens(self._src_lang     )




			def __getstate__(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       self.__dict__.copy()
						UpperCamelCase__					=       None
						return state




			def __setstate__(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       d

						# for backward compatibility
						if not hasattr(self      ,						"sp_model_kwargs"     ):
									UpperCamelCase__					=       {}

						UpperCamelCase__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(self.vocab_file     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       {self.convert_ids_to_tokens(snake_case     ): i for i in range(self.vocab_size     )}
						vocab.update(self.added_tokens_encoder     )
						return vocab




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						return self.sp_model.encode(snake_case      ,						out_type=snake_case     )




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
						UpperCamelCase__					=       self.sp_model.PieceToId(snake_case     )

						# Need to return unknown token if the SP model returned 0
						return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset     )




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       []
						UpperCamelCase__					=       ""
						UpperCamelCase__					=       False
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												if not prev_is_special:
															out_string += " "
												out_string += self.sp_model.decode(snake_case     ) + token
												UpperCamelCase__					=       True
												UpperCamelCase__					=       []
									else:
												current_sub_tokens.append(snake_case     )
												UpperCamelCase__					=       False
						out_string += self.sp_model.decode(snake_case     )
						return out_string.strip()




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						if not os.path.isdir(snake_case     ):
									logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
									return
						UpperCamelCase__					=       os.path.join(
						    snake_case      ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(snake_case     ) and os.path.isfile(self.vocab_file     ):
									copyfile(self.vocab_file      ,						snake_case     )
						elif not os.path.isfile(self.vocab_file     ):
									with open(snake_case      ,						"wb"     ) as fi:
												UpperCamelCase__					=       self.sp_model.serialized_model_proto()
												fi.write(snake_case     )

						return (out_vocab_file,)




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None      ,						snake_case = False     ):

						'''simple docstring'''


						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=snake_case      ,						token_ids_a=snake_case      ,						already_has_special_tokens=snake_case     )

						UpperCamelCase__					=       [1] * len(self.prefix_tokens     )
						UpperCamelCase__					=       [1] * len(self.suffix_tokens     )
						if token_ids_a is None:
									return prefix_ones + ([0] * len(snake_case     )) + suffix_ones
						return prefix_ones + ([0] * len(snake_case     )) + ([0] * len(snake_case     )) + suffix_ones




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						**snake_case     ):

						'''simple docstring'''


						if src_lang is None or tgt_lang is None:
									raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"     )
						UpperCamelCase__					=       src_lang
						UpperCamelCase__					=       self(snake_case      ,						add_special_tokens=snake_case      ,						return_tensors=snake_case      ,						**snake_case     )
						UpperCamelCase__					=       self.convert_tokens_to_ids(snake_case     )
						UpperCamelCase__					=       tgt_lang_id
						return inputs




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = "en_XX"      ,						snake_case = None      ,						snake_case = "ro_RO"      ,						**snake_case      ,						):

						'''simple docstring'''


						UpperCamelCase__					=       src_lang
						UpperCamelCase__					=       tgt_lang
						return super().prepare_seqaseq_batch(snake_case      ,						snake_case      ,						**snake_case     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return self.set_src_lang_special_tokens(self.src_lang     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return self.set_tgt_lang_special_tokens(self.tgt_lang     )




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       self.lang_code_to_id[src_lang]
						UpperCamelCase__					=       [self.cur_lang_code_id]
						UpperCamelCase__					=       [self.eos_token_id]







			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       self.lang_code_to_id[tgt_lang]
						UpperCamelCase__					=       [self.cur_lang_code_id]
						UpperCamelCase__					=       [self.eos_token_id]






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


__UpperCamelCase    		=       {
    'configuration_trocr': ['TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TrOCRConfig'],
    'processing_trocr': ['TrOCRProcessor'],
}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__UpperCamelCase    		=       [
						    'TROCR_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'TrOCRForCausalLM',
						    'TrOCRPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_trocr import TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP, TrOCRConfig
						from .processing_trocr import TrOCRProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_trocr import TROCR_PRETRAINED_MODEL_ARCHIVE_LIST, TrOCRForCausalLM, TrOCRPreTrainedModel

else:
						import sys

						__UpperCamelCase    		=       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


