

'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase__         =	logging.get_logger(__name__)

lowercase__         =	"▁"

lowercase__         =	{"vocab_file": "sentencepiece.bpe.model"}

lowercase__         =	{
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowercase__         =	{
    "facebook/mbart-large-50-one-to-many-mmt": 1024,
}

# fmt: off
lowercase__         =	["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]




class 	A_			(    _snake_case							):






     '''simple docstring'''



     UpperCAmelCase_						:     Dict  		=  VOCAB_FILES_NAMES
     UpperCAmelCase_						:     Union[str, Any]  		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase_						:     Any  		=  PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase_						:     Tuple  		=  ["""input_ids""", """attention_mask"""]

     UpperCAmelCase_						:     List[int]  		=  []
     UpperCAmelCase_						:     List[int]  		=  []


     def __init__(		self			:					Any					,      lowercase_			:					Tuple					,      lowercase_			:					Dict=None					,      lowercase_			:					Tuple=None					,      lowercase_			:					Tuple="</s>"					,      lowercase_			:					Any="</s>"					,      lowercase_			:					Dict="<s>"					,      lowercase_			:					str="<unk>"					,      lowercase_			:					Optional[Any]="<pad>"					,      lowercase_			:					List[Any]="<mask>"					,      lowercase_			:					Optional[Dict[str, Any]] = None					,      **lowercase_			:					Optional[Any]					,      )						->     None:
      # Mask token behave like a normal word, i.e. include the space before it
      UpperCAmelCase	:     Optional[Any]						 =			AddedToken(lowercase_					,      lstrip=lowercase_					,      rstrip=lowercase_	) if isinstance(lowercase_					,      lowercase_	) else mask_token

      UpperCAmelCase	:     str						 =			{} if sp_model_kwargs is None else sp_model_kwargs

      UpperCAmelCase	:     int						 =			kwargs.get('additional_special_tokens'					,      []	)
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=lowercase_					,      tgt_lang=lowercase_					,      eos_token=lowercase_					,      unk_token=lowercase_					,      sep_token=lowercase_					,      cls_token=lowercase_					,      pad_token=lowercase_					,      mask_token=lowercase_					,      sp_model_kwargs=self.sp_model_kwargs					,      **lowercase_					,      )

      UpperCAmelCase	:     int						 =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
      self.sp_model.Load(str(lowercase_	)	)
      UpperCAmelCase	:     Tuple						 =			vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      UpperCAmelCase	:     List[Any]						 =			{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      UpperCAmelCase	:     List[str]						 =			1

      UpperCAmelCase	:     str						 =			len(self.sp_model	)
      UpperCAmelCase	:     List[Any]						 =			{
          code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(lowercase_	)
      }
      UpperCAmelCase	:     Union[str, Any]						 =			{v: k for k, v in self.lang_code_to_id.items()}
      UpperCAmelCase	:     Optional[Any]						 =			len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset

      self.fairseq_tokens_to_ids.update(self.lang_code_to_id	)
      UpperCAmelCase	:     Optional[int]						 =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}

      UpperCAmelCase	:     Union[str, Any]						 =			src_lang if src_lang is not None else 'en_XX'
      UpperCAmelCase	:     Any						 =			self.lang_code_to_id[self._src_lang]
      UpperCAmelCase	:     Dict						 =			tgt_lang
      self.set_src_lang_special_tokens(self._src_lang	)


     @property
     def        UpperCAmelCase_	(		self			:					Optional[Any]	)						->     int:
      return len(self.sp_model	) + len(self.lang_code_to_id	) + self.fairseq_offset + 1  # Plus 1 for the mask token


     @property
     def        UpperCAmelCase_	(		self			:					Any	)						->     str:
      return self._src_lang


     @src_lang.setter
     def        UpperCAmelCase_	(		self			:					Union[str, Any]					,      lowercase_			:					str	)						->     None:
      UpperCAmelCase	:     List[Any]						 =			new_src_lang
      self.set_src_lang_special_tokens(self._src_lang	)


     def __getstate__(		self			:					Union[str, Any]	)						->     Dict:
      UpperCAmelCase	:     Optional[int]						 =			self.__dict__.copy()
      UpperCAmelCase	:     List[str]						 =			None
      return state


     def __setstate__(		self			:					Any					,      lowercase_			:					Dict	)						->     None:
      UpperCAmelCase	:     List[str]						 =			d

      # for backward compatibility
      if not hasattr(self					,      'sp_model_kwargs'	):
       UpperCAmelCase	:     Tuple						 =			{}

      UpperCAmelCase	:     Any						 =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
      self.sp_model.Load(self.vocab_file	)


     def        UpperCAmelCase_	(		self			:					List[str]	)						->     Dict:
      UpperCAmelCase	:     int						 =			{self.convert_ids_to_tokens(lowercase_	): i for i in range(self.vocab_size	)}
      vocab.update(self.added_tokens_encoder	)
      return vocab


     def        UpperCAmelCase_	(		self			:					str					,      lowercase_			:					str	)						->     List[str]:
      return self.sp_model.encode(lowercase_					,      out_type=lowercase_	)


     def        UpperCAmelCase_	(		self			:					int					,      lowercase_			:					str	)						->     int:
      if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
      UpperCAmelCase	:     Tuple						 =			self.sp_model.PieceToId(lowercase_	)

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


     def        UpperCAmelCase_	(		self			:					Union[str, Any]					,      lowercase_			:					int	)						->     str:
      if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset	)


     def        UpperCAmelCase_	(		self			:					Optional[Any]					,      lowercase_			:					int	)						->     Optional[int]:
      UpperCAmelCase	:     List[Any]						 =			[]
      UpperCAmelCase	:     Dict						 =			''
      UpperCAmelCase	:     Union[str, Any]						 =			False
      for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
        if not prev_is_special:
         out_string += " "
        out_string += self.sp_model.decode(lowercase_	) + token
        UpperCAmelCase	:     List[str]						 =			True
        UpperCAmelCase	:     int						 =			[]
       else:
        current_sub_tokens.append(lowercase_	)
        UpperCAmelCase	:     List[Any]						 =			False
      out_string += self.sp_model.decode(lowercase_	)
      return out_string.strip()


     def        UpperCAmelCase_	(		self			:					Optional[int]					,      lowercase_			:					str					,      lowercase_			:					Optional[str] = None	)						->     Tuple[str]:
      if not os.path.isdir(lowercase_	):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
       return
      UpperCAmelCase	:     Optional[Any]						 =			os.path.join(
          lowercase_					,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

      if os.path.abspath(self.vocab_file	) != os.path.abspath(lowercase_	) and os.path.isfile(self.vocab_file	):
       copyfile(self.vocab_file					,      lowercase_	)
      elif not os.path.isfile(self.vocab_file	):
       with open(lowercase_					,      'wb'	) as fi:
        UpperCAmelCase	:     Optional[int]						 =			self.sp_model.serialized_model_proto()
        fi.write(lowercase_	)

      return (out_vocab_file,)


     def        UpperCAmelCase_	(		self			:					Optional[Any]					,      lowercase_			:					List[int]					,      lowercase_			:					Optional[List[int]] = None					,      lowercase_			:					bool = False	)						->     List[int]:

      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=lowercase_					,      token_ids_a=lowercase_					,      already_has_special_tokens=lowercase_	)

      UpperCAmelCase	:     Union[str, Any]						 =			[1] * len(self.prefix_tokens	)
      UpperCAmelCase	:     int						 =			[1] * len(self.suffix_tokens	)
      if token_ids_a is None:
       return prefix_ones + ([0] * len(lowercase_	)) + suffix_ones
      return prefix_ones + ([0] * len(lowercase_	)) + ([0] * len(lowercase_	)) + suffix_ones


     def        UpperCAmelCase_	(		self			:					List[str]					,      lowercase_			:					List[int]					,      lowercase_			:					Optional[List[int]] = None	)						->     List[int]:
      if token_ids_a is None:
       return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


     def        UpperCAmelCase_	(		self			:					Optional[Any]					,      lowercase_			:					List[Any]					,      lowercase_			:					str					,      lowercase_			:					Optional[str]					,      lowercase_			:					Optional[str]					,      **lowercase_			:					Optional[Any]	)						->     Optional[int]:
      if src_lang is None or tgt_lang is None:
       raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
      UpperCAmelCase	:     List[Any]						 =			src_lang
      UpperCAmelCase	:     int						 =			self(lowercase_					,      add_special_tokens=lowercase_					,      return_tensors=lowercase_					,      **lowercase_	)
      UpperCAmelCase	:     Any						 =			self.convert_tokens_to_ids(lowercase_	)
      UpperCAmelCase	:     Dict						 =			tgt_lang_id
      return inputs


     def        UpperCAmelCase_	(		self			:					Union[str, Any]					,      lowercase_			:					List[str]					,      lowercase_			:					str = "en_XX"					,      lowercase_			:					Optional[List[str]] = None					,      lowercase_			:					str = "ro_RO"					,      **lowercase_			:					Dict					,      )						->     BatchEncoding:
      UpperCAmelCase	:     Optional[int]						 =			src_lang
      UpperCAmelCase	:     Optional[Any]						 =			tgt_lang
      return super().prepare_seqaseq_batch(lowercase_					,      lowercase_					,      **lowercase_	)


     def        UpperCAmelCase_	(		self			:					Union[str, Any]	)						->     Union[str, Any]:
      return self.set_src_lang_special_tokens(self.src_lang	)


     def        UpperCAmelCase_	(		self			:					Any	)						->     Optional[int]:
      return self.set_tgt_lang_special_tokens(self.tgt_lang	)


     def        UpperCAmelCase_	(		self			:					Any					,      lowercase_			:					str	)						->     None:
      UpperCAmelCase	:     Union[str, Any]						 =			self.lang_code_to_id[src_lang]
      UpperCAmelCase	:     Optional[Any]						 =			[self.cur_lang_code_id]
      UpperCAmelCase	:     Dict						 =			[self.eos_token_id]




     def        UpperCAmelCase_	(		self			:					Tuple					,      lowercase_			:					str	)						->     None:
      UpperCAmelCase	:     List[str]						 =			self.lang_code_to_id[tgt_lang]
      UpperCAmelCase	:     List[str]						 =			[self.cur_lang_code_id]
      UpperCAmelCase	:     str						 =			[self.eos_token_id]







'''simple docstring'''

from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

lowercase__         =      TypeVar("KEY")
lowercase__         =      TypeVar("VAL")




@dataclass(frozen=_snake_case	,       slots=_snake_case							)
class 	A_			(    Generic[KEY, VAL]							):






     '''simple docstring'''



     UpperCAmelCase_						:     KEY
     UpperCAmelCase_						:     VAL




class 	A_			(    _Item							):






     '''simple docstring'''



     def __init__(		self			:					Any	)						->     None:
      super().__init__(lowercase_					,      lowercase_	)




     def __bool__(		self			:					List[str]	)						->     bool:
      return False



lowercase__         =      _DeletedItem()




class 	A_			(    MutableMapping[KEY, VAL]							):






     '''simple docstring'''



     def __init__(		self			:					Optional[int]					,      lowercase_			:					int = 8					,      lowercase_			:					float = 0.75	)						->     None:
      UpperCAmelCase	:     Dict						 =			initial_block_size
      UpperCAmelCase	:     list[_Item | None]						 =			[None] * initial_block_size
      assert 0.0 < capacity_factor < 1.0
      UpperCAmelCase	:     Any						 =			capacity_factor
      UpperCAmelCase	:     Union[str, Any]						 =			0


     def        UpperCAmelCase_	(		self			:					str					,      lowercase_			:					KEY	)						->     int:
      return hash(lowercase_	) % len(self._buckets	)


     def        UpperCAmelCase_	(		self			:					Any					,      lowercase_			:					int	)						->     int:
      return (ind + 1) % len(self._buckets	)


     def        UpperCAmelCase_	(		self			:					Dict					,      lowercase_			:					int					,      lowercase_			:					KEY					,      lowercase_			:					VAL	)						->     bool:
      UpperCAmelCase	:     List[Any]						 =			self._buckets[ind]
      if not stored:
       UpperCAmelCase	:     Dict						 =			_Item(lowercase_					,      lowercase_	)
       self._len += 1
       return True
      elif stored.key == key:
       UpperCAmelCase	:     Dict						 =			_Item(lowercase_					,      lowercase_	)
       return True
      else:
       return False


     def        UpperCAmelCase_	(		self			:					Any	)						->     bool:
      UpperCAmelCase	:     List[str]						 =			len(self._buckets	) * self._capacity_factor
      return len(self	) >= int(lowercase_	)


     def        UpperCAmelCase_	(		self			:					Optional[int]	)						->     bool:
      if len(self._buckets	) <= self._initial_block_size:
       return False
      UpperCAmelCase	:     Optional[int]						 =			len(self._buckets	) * self._capacity_factor / 2
      return len(self	) < limit


     def        UpperCAmelCase_	(		self			:					Dict					,      lowercase_			:					int	)						->     None:
      UpperCAmelCase	:     int						 =			self._buckets
      UpperCAmelCase	:     List[str]						 =			[None] * new_size
      UpperCAmelCase	:     Dict						 =			0
      for item in old_buckets:
       if item:
        self._add_item(item.key					,      item.val	)


     def        UpperCAmelCase_	(		self			:					Dict	)						->     None:
      self._resize(len(self._buckets	) * 2	)


     def        UpperCAmelCase_	(		self			:					Union[str, Any]	)						->     None:
      self._resize(len(self._buckets	) // 2	)


     def        UpperCAmelCase_	(		self			:					Union[str, Any]					,      lowercase_			:					KEY	)						->     Iterator[int]:
      UpperCAmelCase	:     Dict						 =			self._get_bucket_index(lowercase_	)
      for _ in range(len(self._buckets	)	):
       yield ind
       UpperCAmelCase	:     Union[str, Any]						 =			self._get_next_ind(lowercase_	)


     def        UpperCAmelCase_	(		self			:					Dict					,      lowercase_			:					KEY					,      lowercase_			:					VAL	)						->     None:
      for ind in self._iterate_buckets(lowercase_	):
       if self._try_set(lowercase_					,      lowercase_					,      lowercase_	):
        break


     def __setitem__(		self			:					Union[str, Any]					,      lowercase_			:					KEY					,      lowercase_			:					VAL	)						->     None:
      if self._is_full():
       self._size_up()

      self._add_item(lowercase_					,      lowercase_	)


     def __delitem__(		self			:					Tuple					,      lowercase_			:					KEY	)						->     None:
      for ind in self._iterate_buckets(lowercase_	):
       UpperCAmelCase	:     int						 =			self._buckets[ind]
       if item is None:
        raise KeyError(lowercase_	)
       if item is _deleted:
        continue
       if item.key == key:
        UpperCAmelCase	:     Union[str, Any]						 =			_deleted
        self._len -= 1
        break
      if self._is_sparse():
       self._size_down()


     def __getitem__(		self			:					Dict					,      lowercase_			:					KEY	)						->     VAL:
      for ind in self._iterate_buckets(lowercase_	):
       UpperCAmelCase	:     List[str]						 =			self._buckets[ind]
       if item is None:
        break
       if item is _deleted:
        continue
       if item.key == key:
        return item.val
      raise KeyError(lowercase_	)


     def __len__(		self			:					Optional[Any]	)						->     int:
      return self._len


     def __iter__(		self			:					int	)						->     Iterator[KEY]:
      yield from (item.key for item in self._buckets if item)




     def __repr__(		self			:					Optional[Any]	)						->     str:
      UpperCAmelCase	:     int						 =			' ,'.join(
          f"""{item.key}: {item.val}""" for item in self._buckets if item	)
      return f"""HashMap({val_string})"""




