






"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_	:    Tuple            =      logging.get_logger(__name__)

lowerCAmelCase_	:    Tuple            =      """▁"""

lowerCAmelCase_	:    Optional[Any]            =      {"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase_	:    List[str]            =      {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase_	:    Dict            =      {
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
lowerCAmelCase_	:    int            =      ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]





class   UpperCamelCase_ (       __snake_case  ):

  '''simple docstring'''





  _A       :					Tuple				=	VOCAB_FILES_NAMES
  _A       :					int				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _A       :					Optional[int]				=	PRETRAINED_VOCAB_FILES_MAP
  _A       :					Any				=	['input_ids', 'attention_mask']

  _A       :					List[int]				=	[]
  _A       :					List[int]				=	[]



  def __init__(      self ,		snake_case__ ,		snake_case__="<s>" ,		snake_case__="</s>" ,		snake_case__="</s>" ,		snake_case__="<s>" ,		snake_case__="<unk>" ,		snake_case__="<pad>" ,		snake_case__="<mask>" ,		snake_case__=None ,		snake_case__=None ,		snake_case__=None ,		snake_case__ = None ,		snake_case__=None ,		**snake_case__ ,		)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			AddedToken(A_ ,		lstrip=A_ ,		rstrip=A_				) if isinstance(A_ ,		A_				) else mask_token

         UpperCAmelCase	    =			{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=A_ ,		eos_token=A_ ,		unk_token=A_ ,		sep_token=A_ ,		cls_token=A_ ,		pad_token=A_ ,		mask_token=A_ ,		tokenizer_file=A_ ,		src_lang=A_ ,		tgt_lang=A_ ,		additional_special_tokens=A_ ,		sp_model_kwargs=self.sp_model_kwargs ,		**A_ ,		)

         UpperCAmelCase	    =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.Load(str(A_				)				)
         UpperCAmelCase	    =			vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         UpperCAmelCase	    =			{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         UpperCAmelCase	    =			1

         UpperCAmelCase	    =			len(self.sp_model				)
         UpperCAmelCase	    =			{
             code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(A_				)
         }
         UpperCAmelCase	    =			{v: k for k, v in self.lang_code_to_id.items()}
         UpperCAmelCase	    =			len(self.sp_model				) + len(self.lang_code_to_id				) + self.fairseq_offset

         self.fairseq_tokens_to_ids.update(self.lang_code_to_id				)
         UpperCAmelCase	    =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}
         UpperCAmelCase	    =			list(self.lang_code_to_id.keys()				)

         if additional_special_tokens is not None:
                # Only add those special tokens if they are not already there.
                self._additional_special_tokens.extend(
                    [t for t in additional_special_tokens if t not in self._additional_special_tokens]				)

         UpperCAmelCase	    =			src_lang if src_lang is not None else "en_XX"
         UpperCAmelCase	    =			self.lang_code_to_id[self._src_lang]
         UpperCAmelCase	    =			tgt_lang
         self.set_src_lang_special_tokens(self._src_lang				)



  def __getstate__(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			self.__dict__.copy()
         UpperCAmelCase	    =			None
         UpperCAmelCase	    =			self.sp_model.serialized_model_proto()
         return state



  def __setstate__(      self ,		snake_case__				)			->			Any:



         """simple docstring"""





         UpperCAmelCase	    =			d

         # for backward compatibility
         if not hasattr(self ,		"""sp_model_kwargs"""				):
                UpperCAmelCase	    =			{}

         UpperCAmelCase	    =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto				)



  @property
  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         return len(self.sp_model				) + len(self.lang_code_to_id				) + self.fairseq_offset + 1  # Plus 1 for the mask token



  @property
  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return self._src_lang



  @src_lang.setter
  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Optional[int]:



         """simple docstring"""





         UpperCAmelCase	    =			new_src_lang
         self.set_src_lang_special_tokens(self._src_lang				)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None ,		snake_case__ = False				)			->			List[str]:



         """simple docstring"""





         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=A_ ,		token_ids_a=A_ ,		already_has_special_tokens=A_				)

         UpperCAmelCase	    =			[1] * len(self.prefix_tokens				)
         UpperCAmelCase	    =			[1] * len(self.suffix_tokens				)
         if token_ids_a is None:
                return prefix_ones + ([0] * len(A_				)) + suffix_ones
         return prefix_ones + ([0] * len(A_				)) + ([0] * len(A_				)) + suffix_ones



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			Any:



         """simple docstring"""





         if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			Any:



         """simple docstring"""





         UpperCAmelCase	    =			[self.sep_token_id]
         UpperCAmelCase	    =			[self.cls_token_id]

         if token_ids_a is None:
                return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ ,		snake_case__ ,		**snake_case__				)			->			Any:



         """simple docstring"""





         if src_lang is None or tgt_lang is None:
                raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""				)
         UpperCAmelCase	    =			src_lang
         UpperCAmelCase	    =			self(A_ ,		add_special_tokens=A_ ,		return_tensors=A_ ,		**A_				)
         UpperCAmelCase	    =			self.convert_tokens_to_ids(A_				)
         UpperCAmelCase	    =			tgt_lang_id
         return inputs



  def 		UpperCamelCase_				(      self				)			->			str:



         """simple docstring"""





         UpperCAmelCase	    =			{self.convert_ids_to_tokens(A_				): i for i in range(self.vocab_size				)}
         vocab.update(self.added_tokens_encoder				)
         return vocab



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Dict:



         """simple docstring"""





         return self.sp_model.encode(A_ ,		out_type=A_				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Union[str, Any]:



         """simple docstring"""





         if token in self.fairseq_tokens_to_ids:
                return self.fairseq_tokens_to_ids[token]
         UpperCAmelCase	    =			self.sp_model.PieceToId(A_				)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			List[Any]:



         """simple docstring"""





         if index in self.fairseq_ids_to_tokens:
                return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			"".join(A_				).replace(A_ ,		""" """				).strip()
         return out_string



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			Tuple:



         """simple docstring"""





         if not os.path.isdir(A_				):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
                return
         UpperCAmelCase	    =			os.path.join(
             A_ ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

         if os.path.abspath(self.vocab_file				) != os.path.abspath(A_				) and os.path.isfile(self.vocab_file				):
                copyfile(self.vocab_file ,		A_				)
         elif not os.path.isfile(self.vocab_file				):
                with open(A_ ,		"""wb"""				) as fi:
                       UpperCAmelCase	    =			self.sp_model.serialized_model_proto()
                       fi.write(A_				)

         return (out_vocab_file,)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = "en_XX" ,		snake_case__ = None ,		snake_case__ = "ro_RO" ,		**snake_case__ ,		)			->			int:



         """simple docstring"""





         UpperCAmelCase	    =			src_lang
         UpperCAmelCase	    =			tgt_lang
         return super().prepare_seqaseq_batch(A_ ,		A_ ,		**A_				)



  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return self.set_src_lang_special_tokens(self.src_lang				)



  def 		UpperCamelCase_				(      self				)			->			Any:



         """simple docstring"""





         return self.set_tgt_lang_special_tokens(self.tgt_lang				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			str:



         """simple docstring"""





         UpperCAmelCase	    =			self.lang_code_to_id[src_lang]
         UpperCAmelCase	    =			[]
         UpperCAmelCase	    =			[self.eos_token_id, self.cur_lang_code]



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Optional[int]:



         """simple docstring"""





         UpperCAmelCase	    =			self.lang_code_to_id[lang]
         UpperCAmelCase	    =			[]
         UpperCAmelCase	    =			[self.eos_token_id, self.cur_lang_code]








"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_	:    Any            =      logging.get_logger(__name__)

lowerCAmelCase_	:    Any            =      {
    '''google/canine-s''': '''https://huggingface.co/google/canine-s/resolve/main/config.json''',
    # See all CANINE models at https://huggingface.co/models?filter=canine
}





class   UpperCamelCase_ (       a_  ):
  _A       :					Tuple				=	'canine'



  def __init__(      self ,		snake_case__=7_68 ,		snake_case__=12 ,		snake_case__=12 ,		snake_case__=30_72 ,		snake_case__="gelu" ,		snake_case__=0.1 ,		snake_case__=0.1 ,		snake_case__=1_63_84 ,		snake_case__=16 ,		snake_case__=0.02 ,		snake_case__=1e-12 ,		snake_case__=0 ,		snake_case__=0xE_0_0_0 ,		snake_case__=0xE_0_0_1 ,		snake_case__=4 ,		snake_case__=4 ,		snake_case__=8 ,		snake_case__=1_63_84 ,		snake_case__=1_28 ,		**snake_case__ ,		)			->			int:



         """simple docstring"""





         super().__init__(pad_token_id=snake_case__ ,		bos_token_id=snake_case__ ,		eos_token_id=snake_case__ ,		**snake_case__				)

         UpperCAmelCase	    =			max_position_embeddings
         UpperCAmelCase	    =			hidden_size
         UpperCAmelCase	    =			num_hidden_layers
         UpperCAmelCase	    =			num_attention_heads
         UpperCAmelCase	    =			intermediate_size
         UpperCAmelCase	    =			hidden_act
         UpperCAmelCase	    =			hidden_dropout_prob
         UpperCAmelCase	    =			attention_probs_dropout_prob
         UpperCAmelCase	    =			initializer_range
         UpperCAmelCase	    =			type_vocab_size
         UpperCAmelCase	    =			layer_norm_eps

         # Character config:
         UpperCAmelCase	    =			downsampling_rate
         UpperCAmelCase	    =			upsampling_kernel_size
         UpperCAmelCase	    =			num_hash_functions
         UpperCAmelCase	    =			num_hash_buckets
         UpperCAmelCase	    =			local_transformer_stride
