





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_nllb import NllbTokenizer
else:
      lowercase_          =		None


lowercase_          =		logging.get_logger(__name__)


lowercase_          =		{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowercase_          =		{
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase_          =		{
    """facebook/nllb-large-en-ro""": 1_024,
    """facebook/nllb-200-distilled-600M""": 1_024,
}

# fmt: off
lowercase_          =		["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):
   _UpperCamelCase   :		str			 =	VOCAB_FILES_NAMES
   _UpperCamelCase   :		Union[str, Any]			 =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase   :		Dict			 =	PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase   :		Union[str, Any]			 =	['input_ids', 'attention_mask']
   _UpperCamelCase   :		str			 =	NllbTokenizer

   _UpperCamelCase   :		List[int]			 =	[]
   _UpperCamelCase   :		List[int]			 =	[]


   def __init__(   self      :    Union[str, Any]      ,					a      :    int=None      ,					a      :    Tuple=None      ,					a      :    Optional[int]="<s>"      ,					a      :    List[str]="</s>"      ,					a      :    int="</s>"      ,					a      :    Union[str, Any]="<s>"      ,					a      :    Optional[int]="<unk>"      ,					a      :    str="<pad>"      ,					a      :    Union[str, Any]="<mask>"      ,					a      :    Tuple=None      ,					a      :    Any=None      ,					a      :    Optional[Any]=None      ,					a      :    int=False      ,					**a      :    Dict      ,					)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  AddedToken(a      ,					lstrip=a      ,					rstrip=a	) if isinstance(a      ,					a	) else mask_token
     lowercase__           =  legacy_behaviour
     super().__init__(
         vocab_file=a      ,					tokenizer_file=a      ,					bos_token=a      ,					eos_token=a      ,					sep_token=a      ,					cls_token=a      ,					unk_token=a      ,					pad_token=a      ,					mask_token=a      ,					src_lang=a      ,					tgt_lang=a      ,					additional_special_tokens=a      ,					legacy_behaviour=a      ,					**a      ,					)

     lowercase__           =  vocab_file
     lowercase__           =  False if not self.vocab_file else True

     lowercase__           =  FAIRSEQ_LANGUAGE_CODES.copy()

     if additional_special_tokens is not None:
       # Only add those special tokens if they are not already there.
       _additional_special_tokens.extend(
           [t for t in additional_special_tokens if t not in _additional_special_tokens]	)

     self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}	)
     lowercase__           =  {
         lang_code: self.convert_tokens_to_ids(a	) for lang_code in FAIRSEQ_LANGUAGE_CODES
     }

     lowercase__           =  src_lang if src_lang is not None else 'eng_Latn'
     lowercase__           =  self.convert_tokens_to_ids(self._src_lang	)
     lowercase__           =  tgt_lang
     self.set_src_lang_special_tokens(self._src_lang	)


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    str	)->       str:





     """simple docstring"""







     return self._src_lang


   @src_lang.setter
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict      ,					a      :    str	)->       None:





     """simple docstring"""







     lowercase__           =  new_src_lang
     self.set_src_lang_special_tokens(self._src_lang	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     if token_ids_a is None:
       return self.prefix_tokens + token_ids_a + self.suffix_tokens
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict      ,					a      :    List[int]      ,					a      :    Optional[List[int]] = None	)->       List[int]:





     """simple docstring"""







     lowercase__           =  [self.sep_token_id]
     lowercase__           =  [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep	) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]      ,					a      :    Tuple      ,					a      :    str      ,					a      :    Optional[str]      ,					a      :    Optional[str]      ,					**a      :    Any	)->       str:





     """simple docstring"""







     if src_lang is None or tgt_lang is None:
       raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
     lowercase__           =  src_lang
     lowercase__           =  self(a      ,					add_special_tokens=a      ,					return_tensors=a      ,					**a	)
     lowercase__           =  self.convert_tokens_to_ids(a	)
     lowercase__           =  tgt_lang_id
     return inputs


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict      ,					a      :    List[str]      ,					a      :    str = "eng_Latn"      ,					a      :    Optional[List[str]] = None      ,					a      :    str = "fra_Latn"      ,					**a      :    Tuple      ,					)->       BatchEncoding:





     """simple docstring"""







     lowercase__           =  src_lang
     lowercase__           =  tgt_lang
     return super().prepare_seqaseq_batch(a      ,					a      ,					**a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       List[Any]:





     """simple docstring"""







     return self.set_src_lang_special_tokens(self.src_lang	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       Union[str, Any]:





     """simple docstring"""







     return self.set_tgt_lang_special_tokens(self.tgt_lang	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    int	)->       None:





     """simple docstring"""







     lowercase__           =  self.convert_tokens_to_ids(a	)

     if self.legacy_behaviour:
       lowercase__           =  []
       lowercase__           =  [self.eos_token_id, self.cur_lang_code]
     else:
       lowercase__           =  [self.cur_lang_code]
       lowercase__           =  [self.eos_token_id]

     lowercase__           =  self.convert_ids_to_tokens(self.prefix_tokens	)
     lowercase__           =  self.convert_ids_to_tokens(self.suffix_tokens	)

     lowercase__           =  processors.TemplateProcessing(
         single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,					pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,					self.prefix_tokens + self.suffix_tokens	)	)      ,					)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    str	)->       None:





     """simple docstring"""







     lowercase__           =  self.convert_tokens_to_ids(a	)
     if self.legacy_behaviour:
       lowercase__           =  []
       lowercase__           =  [self.eos_token_id, self.cur_lang_code]
     else:
       lowercase__           =  [self.cur_lang_code]
       lowercase__           =  [self.eos_token_id]

     lowercase__           =  self.convert_ids_to_tokens(self.prefix_tokens	)
     lowercase__           =  self.convert_ids_to_tokens(self.suffix_tokens	)

     lowercase__           =  processors.TemplateProcessing(
         single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,					pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,					self.prefix_tokens + self.suffix_tokens	)	)      ,					)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    str      ,					a      :    Optional[str] = None	)->       Tuple[str]:





     """simple docstring"""







     if not self.can_save_slow_tokenizer:
       raise ValueError(
           'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
           'tokenizer.'	)

     if not os.path.isdir(a	):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""	)
       return
     lowercase__           =  os.path.join(
         a      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

     if os.path.abspath(self.vocab_file	) != os.path.abspath(a	):
       copyfile(self.vocab_file      ,					a	)

     return (out_vocab_file,)









import argparse

import torch
from torch import nn

from transformers import MBartConfig, MBartForConditionalGeneration




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Union[str, Any]:
  lowercase__           =  [
      'encoder.version',
      'decoder.version',
      'model.encoder.version',
      'model.decoder.version',
      '_float_tensor',
      'decoder.output_projection.weight',
  ]
  for k in ignore_keys:
    state_dict.pop(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		)




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Dict:
  lowercase__      ,   lowercase__           =  emb.weight.shape
  lowercase__           =  nn.Linear(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						bias=_SCREAMING_SNAKE_CASE		)
  lowercase__           =  emb.weight.data
  return lin_layer




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE="facebook/mbart-large-en-ro"		,						_SCREAMING_SNAKE_CASE=False		,						_SCREAMING_SNAKE_CASE=False		)    ->       str:
  lowercase__           =  torch.load(_SCREAMING_SNAKE_CASE		,						map_location='cpu'		)['model']
  remove_ignore_keys_(_SCREAMING_SNAKE_CASE		)
  lowercase__           =  state_dict['encoder.embed_tokens.weight'].shape[0]

  lowercase__           =  MBartConfig.from_pretrained(_SCREAMING_SNAKE_CASE		,						vocab_size=_SCREAMING_SNAKE_CASE		)
  if mbart_aa and finetuned:
    lowercase__           =  'relu'

  lowercase__           =  state_dict['decoder.embed_tokens.weight']
  lowercase__           =  MBartForConditionalGeneration(_SCREAMING_SNAKE_CASE		)
  model.model.load_state_dict(_SCREAMING_SNAKE_CASE		)

  if finetuned:
    lowercase__           =  make_linear_from_emb(model.model.shared		)

  return model


if __name__ == "__main__":
      lowercase_          =		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """fairseq_path""", type=str, help="""bart.large, bart.large.cnn or a path to a model.pt on local filesystem."""
      )
      parser.add_argument("""pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
      parser.add_argument(
          """--hf_config""",
          default="""facebook/mbart-large-cc25""",
          type=str,
          help="""Which huggingface architecture to use: mbart-large""",
      )
      parser.add_argument("""--mbart_50""", action="""store_true""", help="""whether the model is mMART-50 checkpoint""")
      parser.add_argument("""--finetuned""", action="""store_true""", help="""whether the model is a fine-tuned checkpoint""")
      lowercase_          =		parser.parse_args()
      lowercase_          =		convert_fairseq_mbart_checkpoint_from_disk(
          args.fairseq_path, hf_config_path=args.hf_config, finetuned=args.finetuned, mbart_aa=args.mbart_aa
      )
      model.save_pretrained(args.pytorch_dump_folder_path)


