

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a_						     =    logging.get_logger(__name__)

a_						     =    '▁'

a_						     =    {'vocab_file': 'sentencepiece.bpe.model'}

a_						     =    {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

a_						     =    {
    'facebook/nllb-200-distilled-600M': 1024,
}

# fmt: off
a_						     =    ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']






class _UpperCamelCase							(							__A   ):
			'''simple docstring'''







			lowerCamelCase__           =VOCAB_FILES_NAMES
			lowerCamelCase__           =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__           =PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__           =['input_ids', 'attention_mask']

			lowerCamelCase__           =[]
			lowerCamelCase__           =[]
			def __init__(			self							: Optional[int]      ,      a							: Dict      ,      a							: Union[str, Any]="<s>"      ,      a							: List[Any]="</s>"      ,      a							: Tuple="</s>"      ,      a							: Tuple="<s>"      ,      a							: Union[str, Any]="<unk>"      ,      a							: int="<pad>"      ,      a							: str="<mask>"      ,      a							: List[Any]=None      ,      a							: Any=None      ,      a							: int=None      ,      a							: Optional[Dict[str, Any]] = None      ,      a							: Optional[Any]=None      ,      a							: List[Any]=False      ,      **a							: Union[str, Any]      ,      )							->       Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[str]				     =       AddedToken(a      ,      lstrip=a      ,      rstrip=a	) if isinstance(a      ,      a	) else mask_token

				SCREAMING_SNAKE_CASE   :      Any				     =       {} if sp_model_kwargs is None else sp_model_kwargs
				SCREAMING_SNAKE_CASE   :      int				     =       legacy_behaviour

				super().__init__(
				    bos_token=a      ,      eos_token=a      ,      unk_token=a      ,      sep_token=a      ,      cls_token=a      ,      pad_token=a      ,      mask_token=a      ,      tokenizer_file=a      ,      src_lang=a      ,      tgt_lang=a      ,      additional_special_tokens=a      ,      sp_model_kwargs=self.sp_model_kwargs      ,      legacy_behaviour=a      ,      **a      ,      )

				SCREAMING_SNAKE_CASE   :      Dict				     =       spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(str(a	)	)
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       vocab_file
				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
				# -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
				# spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

				# Mimic fairseq token-to-id alignment for the first 4 token
				SCREAMING_SNAKE_CASE   :      str				     =       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       1

				SCREAMING_SNAKE_CASE   :      str				     =       len(self.sp_model	)
				SCREAMING_SNAKE_CASE   :      int				     =       {
				    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a	)
				}
				SCREAMING_SNAKE_CASE   :      List[Any]				     =       {v: k for k, v in self.lang_code_to_id.items()}
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset

				self.fairseq_tokens_to_ids.update(self.lang_code_to_id	)
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
				SCREAMING_SNAKE_CASE   :      Any				     =       list(self.lang_code_to_id.keys()	)

				if additional_special_tokens is not None:
					# Only add those special tokens if they are not already there.
					self._additional_special_tokens.extend(
					    [t for t in additional_special_tokens if t not in self._additional_special_tokens]	)

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       src_lang if src_lang is not None else "eng_Latn"
				SCREAMING_SNAKE_CASE   :      str				     =       self.lang_code_to_id[self._src_lang]
				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       tgt_lang
				self.set_src_lang_special_tokens(self._src_lang	)
			def __getstate__(			self							: List[Any]	)							->       Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      str				     =       self.__dict__.copy()
				SCREAMING_SNAKE_CASE   :      str				     =       None
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       self.sp_model.serialized_model_proto()
				return state
			def __setstate__(			self							: List[str]      ,      a							: int	)							->       List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       d

				# for backward compatibility
				if not hasattr(self      ,      "sp_model_kwargs"	):
					SCREAMING_SNAKE_CASE   :      Any				     =       {}

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)
			@property
			def 							__UpperCamelCase (			self							: Tuple	)							->       Optional[int]:

				"""simple docstring"""

				return len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset + 1  # Plus 1 for the mask token
			@property
			def 							__UpperCamelCase (			self							: Any	)							->       str:

				"""simple docstring"""

				return self._src_lang
			@src_lang.setter
			def 							__UpperCamelCase (			self							: Any      ,      a							: str	)							->       None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Tuple				     =       new_src_lang
				self.set_src_lang_special_tokens(self._src_lang	)
			def 							__UpperCamelCase (			self							: Dict      ,      a							: List[int]      ,      a							: Optional[List[int]] = None      ,      a							: bool = False	)							->       List[int]:

				"""simple docstring"""

				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=a      ,      token_ids_a=a      ,      already_has_special_tokens=a	)

				SCREAMING_SNAKE_CASE   :      Dict				     =       [1] * len(self.prefix_tokens	)
				SCREAMING_SNAKE_CASE   :      Dict				     =       [1] * len(self.suffix_tokens	)
				if token_ids_a is None:
					return prefix_ones + ([0] * len(a	)) + suffix_ones
				return prefix_ones + ([0] * len(a	)) + ([0] * len(a	)) + suffix_ones
			def 							__UpperCamelCase (			self							: Dict      ,      a							: List[int]      ,      a							: Optional[List[int]] = None	)							->       List[int]:

				"""simple docstring"""

				if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
			def 							__UpperCamelCase (			self							: List[str]      ,      a							: List[int]      ,      a							: Optional[List[int]] = None	)							->       List[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Dict				     =       [self.sep_token_id]
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       [self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]
			def 							__UpperCamelCase (			self							: Optional[Any]      ,      a							: List[Any]      ,      a							: str      ,      a							: Optional[str]      ,      a							: Optional[str]      ,      **a							: int	)							->       Tuple:

				"""simple docstring"""

				if src_lang is None or tgt_lang is None:
					raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"	)
				SCREAMING_SNAKE_CASE   :      Tuple				     =       src_lang
				SCREAMING_SNAKE_CASE   :      List[Any]				     =       self(a      ,      add_special_tokens=a      ,      return_tensors=a      ,      **a	)
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       self.convert_tokens_to_ids(a	)
				SCREAMING_SNAKE_CASE   :      List[str]				     =       tgt_lang_id
				return inputs
			def 							__UpperCamelCase (			self							: List[str]	)							->       List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       {self.convert_ids_to_tokens(a	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab
			def 							__UpperCamelCase (			self							: Any      ,      a							: str	)							->       List[str]:

				"""simple docstring"""

				return self.sp_model.encode(a      ,      out_type=a	)
			def 							__UpperCamelCase (			self							: Union[str, Any]      ,      a							: int	)							->       List[str]:

				"""simple docstring"""

				if token in self.fairseq_tokens_to_ids:
					return self.fairseq_tokens_to_ids[token]
				SCREAMING_SNAKE_CASE   :      List[Any]				     =       self.sp_model.PieceToId(a	)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
			def 							__UpperCamelCase (			self							: List[Any]      ,      a							: str	)							->       int:

				"""simple docstring"""

				if index in self.fairseq_ids_to_tokens:
					return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset	)
			def 							__UpperCamelCase (			self							: Any      ,      a							: Tuple	)							->       Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      str				     =       "".join(a	).replace(a      ,      " "	).strip()
				return out_string
			def 							__UpperCamelCase (			self							: int      ,      a							: str      ,      a							: Optional[str] = None	)							->       Tuple[str]:

				"""simple docstring"""

				if not os.path.isdir(a	):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
					return
				SCREAMING_SNAKE_CASE   :      List[Any]				     =       os.path.join(
				    a      ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

				if os.path.abspath(self.vocab_file	) != os.path.abspath(a	) and os.path.isfile(self.vocab_file	):
					copyfile(self.vocab_file      ,      a	)
				elif not os.path.isfile(self.vocab_file	):
					with open(a      ,      "wb"	) as fi:
						SCREAMING_SNAKE_CASE   :      List[str]				     =       self.sp_model.serialized_model_proto()
						fi.write(a	)

				return (out_vocab_file,)
			def 							__UpperCamelCase (			self							: Union[str, Any]      ,      a							: List[str]      ,      a							: str = "eng_Latn"      ,      a							: Optional[List[str]] = None      ,      a							: str = "fra_Latn"      ,      **a							: int      ,      )							->       BatchEncoding:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       src_lang
				SCREAMING_SNAKE_CASE   :      Dict				     =       tgt_lang
				return super().prepare_seqaseq_batch(a      ,      a      ,      **a	)
			def 							__UpperCamelCase (			self							: Optional[int]	)							->       Optional[Any]:

				"""simple docstring"""

				return self.set_src_lang_special_tokens(self.src_lang	)
			def 							__UpperCamelCase (			self							: int	)							->       Tuple:

				"""simple docstring"""

				return self.set_tgt_lang_special_tokens(self.tgt_lang	)
			def 							__UpperCamelCase (			self							: Optional[Any]      ,      a							: Optional[int]	)							->       None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[str]				     =       self.lang_code_to_id[src_lang]
				if self.legacy_behaviour:
					SCREAMING_SNAKE_CASE   :      Any				     =       []
					SCREAMING_SNAKE_CASE   :      Any				     =       [self.eos_token_id, self.cur_lang_code]
				else:
					SCREAMING_SNAKE_CASE   :      Any				     =       [self.cur_lang_code]
					SCREAMING_SNAKE_CASE   :      Any				     =       [self.eos_token_id]






			def 							__UpperCamelCase (			self							: int      ,      a							: str	)							->       None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      int				     =       self.lang_code_to_id[lang]
				if self.legacy_behaviour:
					SCREAMING_SNAKE_CASE   :      Optional[int]				     =       []
					SCREAMING_SNAKE_CASE   :      Dict				     =       [self.eos_token_id, self.cur_lang_code]
				else:
					SCREAMING_SNAKE_CASE   :      List[str]				     =       [self.cur_lang_code]
					SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       [self.eos_token_id]


from dataclasses import dataclass
from typing import Optional, Tuple, Union

import flax
import jax.numpy as jnp
from jax import random

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .scheduling_utils_flax import FlaxSchedulerMixin






@flax.struct.dataclass
class _UpperCamelCase							:
			'''simple docstring'''







			lowerCamelCase__           =None
			lowerCamelCase__           =None
			lowerCamelCase__           =None  # sigma(t_i)
			@classmethod
			def 							__UpperCamelCase (			cls							: Tuple	)							->       int:

				"""simple docstring"""

				return cls()






@dataclass
class _UpperCamelCase							(							__A   ):
			'''simple docstring'''







			lowerCamelCase__           =42
			lowerCamelCase__           =42
			lowerCamelCase__           =42






class _UpperCamelCase							(							__A							,	__A   ):
			'''simple docstring'''







			@property
			def 							__UpperCamelCase (			self							: List[str]	)							->       Optional[int]:

				"""simple docstring"""

				return True
			@register_to_config
			def __init__(			self							: List[str]      ,      a							: float = 0.02      ,      a							: float = 100      ,      a							: float = 1.007      ,      a							: float = 80      ,      a							: float = 0.05      ,      a							: float = 50      ,      )							->       Optional[Any]:

				"""simple docstring"""

				pass
			def 							__UpperCamelCase (			self							: Dict	)							->       str:

				"""simple docstring"""

				return KarrasVeSchedulerState.create()
			def 							__UpperCamelCase (			self							: Tuple      ,      a							: KarrasVeSchedulerState      ,      a							: int      ,      a							: Tuple = ()	)							->       KarrasVeSchedulerState:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       jnp.arange(0      ,      a	)[::-1].copy()
				SCREAMING_SNAKE_CASE   :      int				     =       [
				    (
				        self.config.sigma_max**2
				        * (self.config.sigma_min**2 / self.config.sigma_max**2) ** (i / (num_inference_steps - 1))
				    )
				    for i in timesteps
				]

				return state.replace(
				    num_inference_steps=a      ,      schedule=jnp.array(a      ,      dtype=jnp.floataa	)      ,      timesteps=a      ,      )
			def 							__UpperCamelCase (			self							: str      ,      a							: KarrasVeSchedulerState      ,      a							: jnp.ndarray      ,      a							: float      ,      a							: random.KeyArray      ,      )							->       Tuple[jnp.ndarray, float]:

				"""simple docstring"""

				if self.config.s_min <= sigma <= self.config.s_max:
					SCREAMING_SNAKE_CASE   :      Dict				     =       min(self.config.s_churn / state.num_inference_steps      ,      2**0.5 - 1	)
				else:
					SCREAMING_SNAKE_CASE   :      Any				     =       0

				# sample eps ~ N(0, S_noise^2 * I)
				SCREAMING_SNAKE_CASE   :      Any				     =       random.split(a      ,      num=1	)
				SCREAMING_SNAKE_CASE   :      List[str]				     =       self.config.s_noise * random.normal(key=a      ,      shape=sample.shape	)
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       sigma + gamma * sigma
				SCREAMING_SNAKE_CASE   :      str				     =       sample + ((sigma_hat**2 - sigma**2) ** 0.5 * eps)

				return sample_hat, sigma_hat
			def 							__UpperCamelCase (			self							: Union[str, Any]      ,      a							: KarrasVeSchedulerState      ,      a							: jnp.ndarray      ,      a							: float      ,      a							: float      ,      a							: jnp.ndarray      ,      a							: bool = True      ,      )							->       Union[FlaxKarrasVeOutput, Tuple]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      int				     =       sample_hat + sigma_hat * model_output
				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       (sample_hat - pred_original_sample) / sigma_hat
				SCREAMING_SNAKE_CASE   :      str				     =       sample_hat + (sigma_prev - sigma_hat) * derivative

				if not return_dict:
					return (sample_prev, derivative, state)

				return FlaxKarrasVeOutput(prev_sample=a      ,      derivative=a      ,      state=a	)
			def 							__UpperCamelCase (			self							: Optional[int]      ,      a							: KarrasVeSchedulerState      ,      a							: jnp.ndarray      ,      a							: float      ,      a							: float      ,      a							: jnp.ndarray      ,      a							: jnp.ndarray      ,      a							: jnp.ndarray      ,      a							: bool = True      ,      )							->       Union[FlaxKarrasVeOutput, Tuple]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       sample_prev + sigma_prev * model_output
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       (sample_prev - pred_original_sample) / sigma_prev
				SCREAMING_SNAKE_CASE   :      List[Any]				     =       sample_hat + (sigma_prev - sigma_hat) * (0.5 * derivative + 0.5 * derivative_corr)

				if not return_dict:
					return (sample_prev, derivative, state)

				return FlaxKarrasVeOutput(prev_sample=a      ,      derivative=a      ,      state=a	)






			def 							__UpperCamelCase (			self							: int      ,      a							: KarrasVeSchedulerState      ,      a							: Optional[Any]      ,      a							: int      ,      a							: Dict	)							->       List[str]:

				"""simple docstring"""

				raise NotImplementedError()