



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__A       =						logging.get_logger(__name__)

__A       =						'''▁'''

__A       =						{'''vocab_file''': '''sentencepiece.bpe.model'''}

__A       =						{
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

__A       =						{
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
__A       =						['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	a__				: Dict		       =							VOCAB_FILES_NAMES
	a__				: Tuple		       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a__				: Union[str, Any]		       =							PRETRAINED_VOCAB_FILES_MAP
	a__				: Tuple		       =							["input_ids", "attention_mask"]

	a__				: List[int]		       =							[]
	a__				: List[int]		       =							[]




	def __init__(     self     :						List[str]						,  __UpperCAmelCase     :						int						,  __UpperCAmelCase     :						List[str]="<s>"						,  __UpperCAmelCase     :						Optional[Any]="</s>"						,  __UpperCAmelCase     :						Any="</s>"						,  __UpperCAmelCase     :						Any="<s>"						,  __UpperCAmelCase     :						Tuple="<unk>"						,  __UpperCAmelCase     :						Tuple="<pad>"						,  __UpperCAmelCase     :						Optional[int]="<mask>"						,  __UpperCAmelCase     :						List[str]=None						,  __UpperCAmelCase     :						Tuple=None						,  __UpperCAmelCase     :						str=None						,  __UpperCAmelCase     :						Optional[Dict[str, Any]] = None						,  __UpperCAmelCase     :						List[Any]=None						,  **__UpperCAmelCase     :						Optional[Any]						,  )      ->			Union[str, Any]:
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_=       AddedToken(__UpperCAmelCase						,  lstrip=__UpperCAmelCase						,  rstrip=__UpperCAmelCase					) if isinstance(__UpperCAmelCase						,  __UpperCAmelCase					) else mask_token

							UpperCAmelCase_=       {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=__UpperCAmelCase						,  eos_token=__UpperCAmelCase						,  unk_token=__UpperCAmelCase						,  sep_token=__UpperCAmelCase						,  cls_token=__UpperCAmelCase						,  pad_token=__UpperCAmelCase						,  mask_token=__UpperCAmelCase						,  tokenizer_file=__UpperCAmelCase						,  src_lang=__UpperCAmelCase						,  tgt_lang=__UpperCAmelCase						,  additional_special_tokens=__UpperCAmelCase						,  sp_model_kwargs=self.sp_model_kwargs						,  **__UpperCAmelCase						,  )

							UpperCAmelCase_=       spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(str(__UpperCAmelCase					)					)
							UpperCAmelCase_=       vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCAmelCase_=       {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCAmelCase_=       1

							UpperCAmelCase_=       len(self.sp_model					)
							UpperCAmelCase_=       {
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__UpperCAmelCase					)
							}
							UpperCAmelCase_=       {v: k for k, v in self.lang_code_to_id.items()}
							UpperCAmelCase_=       len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
							UpperCAmelCase_=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
							UpperCAmelCase_=       list(self.lang_code_to_id.keys()					)

							if additional_special_tokens is not None:
													# Only add those special tokens if they are not already there.
													self._additional_special_tokens.extend(
													    [t for t in additional_special_tokens if t not in self._additional_special_tokens]					)

							UpperCAmelCase_=       src_lang if src_lang is not None else """en_XX"""
							UpperCAmelCase_=       self.lang_code_to_id[self._src_lang]
							UpperCAmelCase_=       tgt_lang
							self.set_src_lang_special_tokens(self._src_lang					)




	def __getstate__(     self     :						Dict					)      ->			Optional[int]:
							UpperCAmelCase_=       self.__dict__.copy()
							UpperCAmelCase_=       None
							UpperCAmelCase_=       self.sp_model.serialized_model_proto()
							return state




	def __setstate__(     self     :						List[Any]						,  __UpperCAmelCase     :						Optional[int]					)      ->			Any:
							UpperCAmelCase_=       d

							# for backward compatibility
							if not hasattr(self						,  """sp_model_kwargs"""					):
													UpperCAmelCase_=       {}

							UpperCAmelCase_=       spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)




	@property
	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Optional[Any]:
							return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token




	@property
	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			str:
							return self._src_lang




	@src_lang.setter
	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						str					)      ->			None:
							UpperCAmelCase_=       new_src_lang
							self.set_src_lang_special_tokens(self._src_lang					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						List[int]						,  __UpperCAmelCase     :						Optional[List[int]] = None						,  __UpperCAmelCase     :						bool = False					)      ->			List[int]:

							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=__UpperCAmelCase						,  token_ids_a=__UpperCAmelCase						,  already_has_special_tokens=__UpperCAmelCase					)

							UpperCAmelCase_=       [1] * len(self.prefix_tokens					)
							UpperCAmelCase_=       [1] * len(self.suffix_tokens					)
							if token_ids_a is None:
													return prefix_ones + ([0] * len(__UpperCAmelCase					)) + suffix_ones
							return prefix_ones + ([0] * len(__UpperCAmelCase					)) + ([0] * len(__UpperCAmelCase					)) + suffix_ones




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						List[int]						,  __UpperCAmelCase     :						Optional[List[int]] = None					)      ->			List[int]:
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						List[int]						,  __UpperCAmelCase     :						Optional[List[int]] = None					)      ->			List[int]:
							UpperCAmelCase_=       [self.sep_token_id]
							UpperCAmelCase_=       [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						Dict						,  __UpperCAmelCase     :						str						,  __UpperCAmelCase     :						Optional[str]						,  __UpperCAmelCase     :						Optional[str]						,  **__UpperCAmelCase     :						Any					)      ->			Tuple:
							if src_lang is None or tgt_lang is None:
													raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""					)
							UpperCAmelCase_=       src_lang
							UpperCAmelCase_=       self(__UpperCAmelCase						,  add_special_tokens=__UpperCAmelCase						,  return_tensors=__UpperCAmelCase						,  **__UpperCAmelCase					)
							UpperCAmelCase_=       self.convert_tokens_to_ids(__UpperCAmelCase					)
							UpperCAmelCase_=       tgt_lang_id
							return inputs




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			int:
							UpperCAmelCase_=       {self.convert_ids_to_tokens(__UpperCAmelCase					): i for i in range(self.vocab_size					)}
							vocab.update(self.added_tokens_encoder					)
							return vocab




	def   _SCREAMING_SNAKE_CASE		(     self     :						int						,  __UpperCAmelCase     :						str					)      ->			List[str]:
							return self.sp_model.encode(__UpperCAmelCase						,  out_type=__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]						,  __UpperCAmelCase     :						Any					)      ->			Optional[Any]:
							if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
							UpperCAmelCase_=       self.sp_model.PieceToId(__UpperCAmelCase					)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						List[Any]					)      ->			List[Any]:
							if index in self.fairseq_ids_to_tokens:
													return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Tuple						,  __UpperCAmelCase     :						int					)      ->			Union[str, Any]:
							UpperCAmelCase_=       """""".join(__UpperCAmelCase					).replace(__UpperCAmelCase						,  """ """					).strip()
							return out_string




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						str						,  __UpperCAmelCase     :						Optional[str] = None					)      ->			Tuple[str]:
							if not os.path.isdir(__UpperCAmelCase					):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
													return
							UpperCAmelCase_=       os.path.join(
							    __UpperCAmelCase						,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(__UpperCAmelCase					) and os.path.isfile(self.vocab_file					):
													copyfile(self.vocab_file						,  __UpperCAmelCase					)
							elif not os.path.isfile(self.vocab_file					):
													with open(__UpperCAmelCase						,  """wb"""					) as fi:
																			UpperCAmelCase_=       self.sp_model.serialized_model_proto()
																			fi.write(__UpperCAmelCase					)

							return (out_vocab_file,)




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						List[str]						,  __UpperCAmelCase     :						str = "en_XX"						,  __UpperCAmelCase     :						Optional[List[str]] = None						,  __UpperCAmelCase     :						str = "ro_RO"						,  **__UpperCAmelCase     :						Dict						,  )      ->			BatchEncoding:
							UpperCAmelCase_=       src_lang
							UpperCAmelCase_=       tgt_lang
							return super().prepare_seqaseq_batch(__UpperCAmelCase						,  __UpperCAmelCase						,  **__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			Optional[Any]:
							return self.set_src_lang_special_tokens(self.src_lang					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Any:
							return self.set_tgt_lang_special_tokens(self.tgt_lang					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]						,  __UpperCAmelCase     :						str					)      ->			None:
							UpperCAmelCase_=       self.lang_code_to_id[src_lang]
							UpperCAmelCase_=       []
							UpperCAmelCase_=       [self.eos_token_id, self.cur_lang_code]




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						str					)      ->			None:
							UpperCAmelCase_=       self.lang_code_to_id[lang]
							UpperCAmelCase_=       []
							UpperCAmelCase_=       [self.eos_token_id, self.cur_lang_code]









__A       =						6_5521



def 	__a					(   lowerCAmelCase_       : str    )				->			int:






						'''simple docstring'''



						UpperCAmelCase_=       1
						UpperCAmelCase_=       0
						for plain_chr in plain_text:
												UpperCAmelCase_=       (a + ord(lowerCAmelCase_    )) % MOD_ADLER
												UpperCAmelCase_=       (b + a) % MOD_ADLER
						return (b << 16) | a




