






"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_nllb import NllbTokenizer
else:
 _lowerCAmelCase	:	str				=      None


_lowerCAmelCase	:	Dict				=      logging.get_logger(__name__)


_lowerCAmelCase	:	List[str]				=      {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_lowerCAmelCase	:	Union[str, Any]				=      {
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

_lowerCAmelCase	:	int				=      {
    """facebook/nllb-large-en-ro""": 1_024,
    """facebook/nllb-200-distilled-600M""": 1_024,
}

# fmt: off
_lowerCAmelCase	:	Union[str, Any]				=      ["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]




class 			lowerCAmelCase__			(    __magic_name__   ):
   SCREAMING_SNAKE_CASE_    =VOCAB_FILES_NAMES
   SCREAMING_SNAKE_CASE_    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   SCREAMING_SNAKE_CASE_    =PRETRAINED_VOCAB_FILES_MAP
   SCREAMING_SNAKE_CASE_    =['''input_ids''', '''attention_mask''']
   SCREAMING_SNAKE_CASE_    =NllbTokenizer

   SCREAMING_SNAKE_CASE_    =[]
   SCREAMING_SNAKE_CASE_    =[]
   def __init__(		self      :	int							,			snake_case__      :	Optional[int]=None							,			snake_case__      :	Dict=None							,			snake_case__      :	List[Any]="<s>"							,			snake_case__      :	Union[str, Any]="</s>"							,			snake_case__      :	Dict="</s>"							,			snake_case__      :	Union[str, Any]="<s>"							,			snake_case__      :	Optional[int]="<unk>"							,			snake_case__      :	List[Any]="<pad>"							,			snake_case__      :	Dict="<mask>"							,			snake_case__      :	Dict=None							,			snake_case__      :	Tuple=None							,			snake_case__      :	Union[str, Any]=None							,			snake_case__      :	Optional[Any]=False							,			**snake_case__      :	List[str]							,			):


        '''simple docstring'''
        # Mask token behave like a normal word, i.e. include the space before it
        UpperCAmelCase__						:    Any							=							AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__							) if isinstance(snake_case__							,			snake_case__							) else mask_token
        UpperCAmelCase__						:    Dict							=							legacy_behaviour
        super().__init__(
            vocab_file=snake_case__							,			tokenizer_file=snake_case__							,			bos_token=snake_case__							,			eos_token=snake_case__							,			sep_token=snake_case__							,			cls_token=snake_case__							,			unk_token=snake_case__							,			pad_token=snake_case__							,			mask_token=snake_case__							,			src_lang=snake_case__							,			tgt_lang=snake_case__							,			additional_special_tokens=snake_case__							,			legacy_behaviour=snake_case__							,			**snake_case__							,			)

        UpperCAmelCase__						:    Any							=							vocab_file
        UpperCAmelCase__						:    int							=							False if not self.vocab_file else True

        UpperCAmelCase__						:    Optional[Any]							=							FAIRSEQ_LANGUAGE_CODES.copy()

        if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             _additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in _additional_special_tokens]							)

        self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}							)
        UpperCAmelCase__						:    List[str]							=							{
            lang_code: self.convert_tokens_to_ids(snake_case__							) for lang_code in FAIRSEQ_LANGUAGE_CODES
        }

        UpperCAmelCase__						:    List[Any]							=							src_lang if src_lang is not None else "eng_Latn"
        UpperCAmelCase__						:    List[str]							=							self.convert_tokens_to_ids(self._src_lang							)
        UpperCAmelCase__						:    List[Any]							=							tgt_lang
        self.set_src_lang_special_tokens(self._src_lang							)
   @property
   def       __a							(		self      :	Optional[int]							):


        '''simple docstring'''
        return self._src_lang
   @src_lang.setter
   def       __a							(		self      :	Dict							,			snake_case__      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							new_src_lang
        self.set_src_lang_special_tokens(self._src_lang							)
   def       __a							(		self      :	Dict							,			snake_case__      :	List[int]							,			snake_case__      :	Optional[List[int]] = None							):


        '''simple docstring'''
        if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
   def       __a							(		self      :	Any							,			snake_case__      :	List[int]							,			snake_case__      :	Optional[List[int]] = None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							[self.sep_token_id]
        UpperCAmelCase__						:    str							=							[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep							) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]
   def       __a							(		self      :	Tuple							,			snake_case__      :	List[Any]							,			snake_case__      :	str							,			snake_case__      :	Optional[str]							,			snake_case__      :	Optional[str]							,			**snake_case__      :	Union[str, Any]							):


        '''simple docstring'''
        if src_lang is None or tgt_lang is None:
             raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"							)
        UpperCAmelCase__						:    int							=							src_lang
        UpperCAmelCase__						:    Dict							=							self(snake_case__							,			add_special_tokens=snake_case__							,			return_tensors=snake_case__							,			**snake_case__							)
        UpperCAmelCase__						:    Tuple							=							self.convert_tokens_to_ids(snake_case__							)
        UpperCAmelCase__						:    Any							=							tgt_lang_id
        return inputs
   def       __a							(		self      :	List[str]							,			snake_case__      :	List[str]							,			snake_case__      :	str = "eng_Latn"							,			snake_case__      :	Optional[List[str]] = None							,			snake_case__      :	str = "fra_Latn"							,			**snake_case__      :	Tuple							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							src_lang
        UpperCAmelCase__						:    Optional[Any]							=							tgt_lang
        return super().prepare_seqaseq_batch(snake_case__							,			snake_case__							,			**snake_case__							)
   def       __a							(		self      :	int							):


        '''simple docstring'''
        return self.set_src_lang_special_tokens(self.src_lang							)
   def       __a							(		self      :	Dict							):


        '''simple docstring'''
        return self.set_tgt_lang_special_tokens(self.tgt_lang							)
   def       __a							(		self      :	List[str]							,			snake_case__      :	Optional[Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Any							=							self.convert_tokens_to_ids(snake_case__							)

        if self.legacy_behaviour:
             UpperCAmelCase__						:    Tuple							=							[]
             UpperCAmelCase__						:    Optional[Any]							=							[self.eos_token_id, self.cur_lang_code]
        else:
             UpperCAmelCase__						:    Tuple							=							[self.cur_lang_code]
             UpperCAmelCase__						:    Any							=							[self.eos_token_id]

        UpperCAmelCase__						:    Optional[int]							=							self.convert_ids_to_tokens(self.prefix_tokens							)
        UpperCAmelCase__						:    Any							=							self.convert_ids_to_tokens(self.suffix_tokens							)

        UpperCAmelCase__						:    Optional[Any]							=							processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,			self.prefix_tokens + self.suffix_tokens							)							)							,			)
   def       __a							(		self      :	int							,			snake_case__      :	str							):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							self.convert_tokens_to_ids(snake_case__							)
        if self.legacy_behaviour:
             UpperCAmelCase__						:    Union[str, Any]							=							[]
             UpperCAmelCase__						:    Any							=							[self.eos_token_id, self.cur_lang_code]
        else:
             UpperCAmelCase__						:    Tuple							=							[self.cur_lang_code]
             UpperCAmelCase__						:    Optional[Any]							=							[self.eos_token_id]

        UpperCAmelCase__						:    Optional[Any]							=							self.convert_ids_to_tokens(self.prefix_tokens							)
        UpperCAmelCase__						:    Optional[int]							=							self.convert_ids_to_tokens(self.suffix_tokens							)

        UpperCAmelCase__						:    Optional[int]							=							processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str							,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str							,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str							,			self.prefix_tokens + self.suffix_tokens							)							)							,			)




   def       __a							(		self      :	Union[str, Any]							,			snake_case__      :	str							,			snake_case__      :	Optional[str] = None							):


        '''simple docstring'''
        if not self.can_save_slow_tokenizer:
             raise ValueError(
                 "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                 "tokenizer."							)

        if not os.path.isdir(snake_case__							):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory.'							)
             return
        UpperCAmelCase__						:    Union[str, Any]							=							os.path.join(
            snake_case__							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

        if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case__							):
             copyfile(self.vocab_file							,			snake_case__							)

        return (out_vocab_file,)










"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase	:	Tuple				=      {
    """configuration_lxmert""": ["""LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LxmertConfig"""],
    """tokenization_lxmert""": ["""LxmertTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	int				=      ["""LxmertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Tuple				=      [
     """LxmertEncoder""",
     """LxmertForPreTraining""",
     """LxmertForQuestionAnswering""",
     """LxmertModel""",
     """LxmertPreTrainedModel""",
     """LxmertVisualFeatureEncoder""",
     """LxmertXLayer""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Optional[int]				=      [
     """TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFLxmertForPreTraining""",
     """TFLxmertMainLayer""",
     """TFLxmertModel""",
     """TFLxmertPreTrainedModel""",
     """TFLxmertVisualFeatureEncoder""",
 ]


if TYPE_CHECKING:
 from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
 from .tokenization_lxmert import LxmertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_lxmert_fast import LxmertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_lxmert import (
      LxmertEncoder,
      LxmertForPreTraining,
      LxmertForQuestionAnswering,
      LxmertModel,
      LxmertPreTrainedModel,
      LxmertVisualFeatureEncoder,
      LxmertXLayer,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_lxmert import (
      TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLxmertForPreTraining,
      TFLxmertMainLayer,
      TFLxmertModel,
      TFLxmertPreTrainedModel,
      TFLxmertVisualFeatureEncoder,
  )

else:
 import sys

 _lowerCAmelCase	:	Any				=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


