

'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE     =						logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     =						"""▁"""

__SCREAMING_SNAKE_CASE     =						{"""vocab_file""": """sentencepiece.bpe.model"""}

__SCREAMING_SNAKE_CASE     =						{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

__SCREAMING_SNAKE_CASE     =						{
    """facebook/mbart-large-50-one-to-many-mmt""": 1_0_2_4,
}

# fmt: off
__SCREAMING_SNAKE_CASE     =						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]






class 			lowerCAmelCase__     (					lowerCamelCase__						):




     """simple docstring"""





     __UpperCamelCase									= VOCAB_FILES_NAMES
     __UpperCamelCase									= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase									= PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase									= ["input_ids", "attention_mask"]

     __UpperCamelCase									= []
     __UpperCamelCase									= []







     def __init__(		self						:	List[Any]      ,						A__						:	Dict      ,						A__						:	List[str]=None      ,						A__						:	List[Any]=None      ,						A__						:	str="</s>"      ,						A__						:	Dict="</s>"      ,						A__						:	Any="<s>"      ,						A__						:	int="<unk>"      ,						A__						:	Optional[Any]="<pad>"      ,						A__						:	Dict="<mask>"      ,						A__						:	Optional[int] = None      ,						**A__						:	List[str]      ,						)       ->			None:
          '''simple docstring'''




          a__   :      Tuple     =						AddedToken(SCREAMING_SNAKE_CASE__      ,						lstrip=SCREAMING_SNAKE_CASE__      ,						rstrip=SCREAMING_SNAKE_CASE__	) if isinstance(SCREAMING_SNAKE_CASE__      ,						SCREAMING_SNAKE_CASE__	) else mask_token

          a__   :      List[Any]     =						{} if sp_model_kwargs is None else sp_model_kwargs

          a__   :      str     =						kwargs.get('''additional_special_tokens'''      ,						[]	)
          kwargs["additional_special_tokens"] += [
              code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              src_lang=SCREAMING_SNAKE_CASE__      ,						tgt_lang=SCREAMING_SNAKE_CASE__      ,						eos_token=SCREAMING_SNAKE_CASE__      ,						unk_token=SCREAMING_SNAKE_CASE__      ,						sep_token=SCREAMING_SNAKE_CASE__      ,						cls_token=SCREAMING_SNAKE_CASE__      ,						pad_token=SCREAMING_SNAKE_CASE__      ,						mask_token=SCREAMING_SNAKE_CASE__      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**SCREAMING_SNAKE_CASE__      ,						)

          a__   :      int     =						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(str(SCREAMING_SNAKE_CASE__	)	)
          a__   :      str     =						vocab_file

          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
          # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
          # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

          # Mimic fairseq token-to-id alignment for the first 4 token
          a__   :      Optional[int]     =						{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          a__   :      Any     =						1

          a__   :      Optional[Any]     =						len(self.sp_model	)
          a__   :      Any     =						{
              code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(SCREAMING_SNAKE_CASE__	)
          }
          a__   :      Union[str, Any]     =						{v: k for k, v in self.lang_code_to_id.items()}
          a__   :      Optional[Any]     =						len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset

          self.fairseq_tokens_to_ids.update(self.lang_code_to_id	)
          a__   :      Tuple     =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}

          a__   :      int     =						src_lang if src_lang is not None else '''en_XX'''
          a__   :      int     =						self.lang_code_to_id[self._src_lang]
          a__   :      int     =						tgt_lang
          self.set_src_lang_special_tokens(self._src_lang	)







     @property
     def 			__lowerCAmelCase	(		self						:	List[str]	)       ->			int:
          '''simple docstring'''




          return len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset + 1  # Plus 1 for the mask token







     @property
     def 			__lowerCAmelCase	(		self						:	Dict	)       ->			str:
          '''simple docstring'''




          return self._src_lang







     @src_lang.setter
     def 			__lowerCAmelCase	(		self						:	int      ,						A__						:	int	)       ->			None:
          '''simple docstring'''




          a__   :      int     =						new_src_lang
          self.set_src_lang_special_tokens(self._src_lang	)







     def __getstate__(		self						:	List[str]	)       ->			Dict:
          '''simple docstring'''




          a__   :      int     =						self.__dict__.copy()
          a__   :      Any     =						None
          return state







     def __setstate__(		self						:	int      ,						A__						:	Union[str, Any]	)       ->			None:
          '''simple docstring'''




          a__   :      List[str]     =						d

          # for backward compatibility
          if not hasattr(self      ,						'''sp_model_kwargs'''	):
               a__   :      int     =						{}

          a__   :      int     =						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
          self.sp_model.Load(self.vocab_file	)







     def 			__lowerCAmelCase	(		self						:	Any	)       ->			Dict:
          '''simple docstring'''




          a__   :      Any     =						{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__	): i for i in range(self.vocab_size	)}
          vocab.update(self.added_tokens_encoder	)
          return vocab







     def 			__lowerCAmelCase	(		self						:	Optional[Any]      ,						A__						:	str	)       ->			List[str]:
          '''simple docstring'''




          return self.sp_model.encode(SCREAMING_SNAKE_CASE__      ,						out_type=SCREAMING_SNAKE_CASE__	)







     def 			__lowerCAmelCase	(		self						:	Tuple      ,						A__						:	int	)       ->			int:
          '''simple docstring'''




          if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
          a__   :      Any     =						self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__	)

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







     def 			__lowerCAmelCase	(		self						:	Tuple      ,						A__						:	Dict	)       ->			str:
          '''simple docstring'''




          if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset	)







     def 			__lowerCAmelCase	(		self						:	str      ,						A__						:	int	)       ->			str:
          '''simple docstring'''




          a__   :      Any     =						[]
          a__   :      List[str]     =						''''''
          a__   :      Any     =						False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__	) + token
                    a__   :      Union[str, Any]     =						True
                    a__   :      Dict     =						[]
               else:
                    current_sub_tokens.append(SCREAMING_SNAKE_CASE__	)
                    a__   :      Dict     =						False
          out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__	)
          return out_string.strip()







     def 			__lowerCAmelCase	(		self						:	List[Any]      ,						A__						:	Optional[Any]      ,						A__						:	List[str] = None	)       ->			Tuple[str]:
          '''simple docstring'''




          if not os.path.isdir(SCREAMING_SNAKE_CASE__	):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
               return
          a__   :      Optional[Any]     =						os.path.join(
              SCREAMING_SNAKE_CASE__      ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

          if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE__	) and os.path.isfile(self.vocab_file	):
               copyfile(self.vocab_file      ,						SCREAMING_SNAKE_CASE__	)
          elif not os.path.isfile(self.vocab_file	):
               with open(SCREAMING_SNAKE_CASE__      ,						'''wb'''	) as fi:
                    a__   :      Optional[Any]     =						self.sp_model.serialized_model_proto()
                    fi.write(SCREAMING_SNAKE_CASE__	)

          return (out_vocab_file,)







     def 			__lowerCAmelCase	(		self						:	Dict      ,						A__						:	str      ,						A__						:	str = None      ,						A__						:	Optional[int] = False	)       ->			List[int]:
          '''simple docstring'''




          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=SCREAMING_SNAKE_CASE__      ,						token_ids_a=SCREAMING_SNAKE_CASE__      ,						already_has_special_tokens=SCREAMING_SNAKE_CASE__	)

          a__   :      List[str]     =						[1] * len(self.prefix_tokens	)
          a__   :      Tuple     =						[1] * len(self.suffix_tokens	)
          if token_ids_a is None:
               return prefix_ones + ([0] * len(SCREAMING_SNAKE_CASE__	)) + suffix_ones
          return prefix_ones + ([0] * len(SCREAMING_SNAKE_CASE__	)) + ([0] * len(SCREAMING_SNAKE_CASE__	)) + suffix_ones







     def 			__lowerCAmelCase	(		self						:	List[str]      ,						A__						:	Tuple      ,						A__						:	Tuple = None	)       ->			List[int]:
          '''simple docstring'''




          if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def 			__lowerCAmelCase	(		self						:	Optional[Any]      ,						A__						:	Dict      ,						A__						:	Tuple      ,						A__						:	Union[str, Any]      ,						A__						:	str      ,						**A__						:	Any	)       ->			List[str]:
          '''simple docstring'''




          if src_lang is None or tgt_lang is None:
               raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''	)
          a__   :      int     =						src_lang
          a__   :      Dict     =						self(SCREAMING_SNAKE_CASE__      ,						add_special_tokens=SCREAMING_SNAKE_CASE__      ,						return_tensors=SCREAMING_SNAKE_CASE__      ,						**SCREAMING_SNAKE_CASE__	)
          a__   :      Tuple     =						self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__	)
          a__   :      Optional[int]     =						tgt_lang_id
          return inputs







     def 			__lowerCAmelCase	(		self						:	Union[str, Any]      ,						A__						:	Dict      ,						A__						:	Optional[Any] = "en_XX"      ,						A__						:	List[Any] = None      ,						A__						:	Optional[Any] = "ro_RO"      ,						**A__						:	Tuple      ,						)       ->			BatchEncoding:
          '''simple docstring'''




          a__   :      Dict     =						src_lang
          a__   :      Union[str, Any]     =						tgt_lang
          return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE__      ,						SCREAMING_SNAKE_CASE__      ,						**SCREAMING_SNAKE_CASE__	)







     def 			__lowerCAmelCase	(		self						:	Optional[Any]	)       ->			List[Any]:
          '''simple docstring'''




          return self.set_src_lang_special_tokens(self.src_lang	)







     def 			__lowerCAmelCase	(		self						:	Dict	)       ->			str:
          '''simple docstring'''




          return self.set_tgt_lang_special_tokens(self.tgt_lang	)







     def 			__lowerCAmelCase	(		self						:	List[Any]      ,						A__						:	Tuple	)       ->			None:
          '''simple docstring'''




          a__   :      Tuple     =						self.lang_code_to_id[src_lang]
          a__   :      Optional[int]     =						[self.cur_lang_code_id]
          a__   :      Optional[int]     =						[self.eos_token_id]







     def 			__lowerCAmelCase	(		self						:	str      ,						A__						:	Dict	)       ->			None:
          '''simple docstring'''




          a__   :      Dict     =						self.lang_code_to_id[tgt_lang]
          a__   :      Tuple     =						[self.cur_lang_code_id]
          a__   :      Optional[int]     =						[self.eos_token_id]




"""simple docstring"""






from collections.abc import Callable

import numpy as np





def 				A_       ( snake_case__       ,					snake_case__       ,					snake_case__       ,					snake_case__       ,					snake_case__		)					->  np.ndarray:
			_UpperCamelCase					:str					     =	int(np.ceil((x_end - xa) / step_size		)		)
			_UpperCamelCase					:Union[str, Any]					     =	np.zeros((n + 1,)		)
			_UpperCamelCase					:List[str]					     =	ya
			_UpperCamelCase					:Any					     =	xa

			for k in range(snake_case__		):
						_UpperCamelCase					:Union[str, Any]					     =	y[k] + step_size * ode_func(snake_case__       ,					y[k]		)
						x += step_size

			return y


if __name__ == "__main__":
					import doctest

					doctest.testmod()

