


'''simple docstring'''







import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


lowerCAmelCase			:   Any 		=  logging.get_logger(__name__)

lowerCAmelCase			:   Tuple 		=  {
    """facebook/musicgen-small""": """https://huggingface.co/facebook/musicgen-small/resolve/main/config.json""",
    # See all Musicgen models at https://huggingface.co/models?filter=musicgen
}
class 		UpperCamelCase__    (       _UpperCamelCase		):



     """simple docstring"""






     __magic_name__				=			"musicgen_decoder"
     __magic_name__				=			["past_key_values"]







     def __init__(       self					,		snake_case__=2048					,		snake_case__=2048					,		snake_case__=24					,		snake_case__=4096					,		snake_case__=16					,		snake_case__=0.0					,		snake_case__=True					,		snake_case__="gelu"					,		snake_case__=1024					,		snake_case__=0.1					,		snake_case__=0.0					,		snake_case__=0.0					,		snake_case__=0.02					,		snake_case__=False					,		snake_case__=4					,		snake_case__=2048					,		snake_case__=2048					,		snake_case__=None					,		snake_case__=False					,		**snake_case__					,		):

            '''simple docstring'''

            _lowerCAmelCase			:							str      	=				vocab_size
            _lowerCAmelCase			:							str      	=				max_position_embeddings
            _lowerCAmelCase			:							int      	=				hidden_size
            _lowerCAmelCase			:							Any      	=				ffn_dim
            _lowerCAmelCase			:							str      	=				num_hidden_layers
            _lowerCAmelCase			:							Any      	=				num_attention_heads
            _lowerCAmelCase			:							str      	=				dropout
            _lowerCAmelCase			:							Any      	=				attention_dropout
            _lowerCAmelCase			:							List[str]      	=				activation_dropout
            _lowerCAmelCase			:							Optional[Any]      	=				activation_function
            _lowerCAmelCase			:							List[str]      	=				initializer_factor
            _lowerCAmelCase			:							Union[str, Any]      	=				layerdrop
            _lowerCAmelCase			:							Any      	=				use_cache
            _lowerCAmelCase			:							Optional[int]      	=				scale_embedding  # scale factor will be sqrt(d_model) if True
            _lowerCAmelCase			:							int      	=				num_codebooks
            super().__init__(
                pad_token_id=__a					,		bos_token_id=__a					,		eos_token_id=__a					,		tie_word_embeddings=__a					,		**__a					,		)
class 		UpperCamelCase__    (       _UpperCamelCase		):



     """simple docstring"""






     __magic_name__				=			"musicgen"
     __magic_name__				=			True







     def __init__(       self					,		**snake_case__      ):

            '''simple docstring'''

            super().__init__(**__a      )
            if "text_encoder" not in kwargs or "audio_encoder" not in kwargs or "decoder" not in kwargs:
                   raise ValueError('Config has to be initialized with text_encoder, audio_encoder and decoder config'      )

            _lowerCAmelCase			:							List[str]      	=				kwargs.pop('text_encoder'      )
            _lowerCAmelCase			:							Tuple      	=				text_encoder_config.pop('model_type'      )

            _lowerCAmelCase			:							Optional[int]      	=				kwargs.pop('audio_encoder'      )
            _lowerCAmelCase			:							List[str]      	=				audio_encoder_config.pop('model_type'      )

            _lowerCAmelCase			:							List[Any]      	=				kwargs.pop('decoder'      )

            _lowerCAmelCase			:							Tuple      	=				AutoConfig.for_model(__a					,		**__a      )
            _lowerCAmelCase			:							Dict      	=				AutoConfig.for_model(__a					,		**__a      )
            _lowerCAmelCase			:							List[Any]      	=				MusicgenDecoderConfig(**__a      )
            _lowerCAmelCase			:							Tuple      	=				True







     @classmethod
     def a		(       cls					,		snake_case__					,		snake_case__					,		snake_case__					,		**snake_case__					,		):

            '''simple docstring'''

            return cls(
                text_encoder=text_encoder_config.to_dict()					,		audio_encoder=audio_encoder_config.to_dict()					,		decoder=decoder_config.to_dict()					,		**__a					,		)







     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							Any      	=				copy.deepcopy(self.__dict__      )
            _lowerCAmelCase			:							Any      	=				self.text_encoder.to_dict()
            _lowerCAmelCase			:							List[str]      	=				self.audio_encoder.to_dict()
            _lowerCAmelCase			:							Dict      	=				self.decoder.to_dict()
            _lowerCAmelCase			:							Tuple      	=				self.__class__.model_type
            return output








"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  lowerCamelCase__      =      None


lowerCamelCase__      =      logging.get_logger(__name__)

lowerCamelCase__      =      {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase__      =      {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

lowerCamelCase__      =      {
    "facebook/mbart-large-50-one-to-many-mmt": 1024,
}

# fmt: off
lowerCamelCase__      =      ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]



class    __SCREAMING_SNAKE_CASE			(  _UpperCamelCase      ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__							:Tuple		   =    VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__							:List[Any]		   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__							:Union[str, Any]		   =    PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__							:int		   =    ["input_ids", "attention_mask"]
  SCREAMING_SNAKE_CASE__							:Dict		   =    MBartaaTokenizer

  SCREAMING_SNAKE_CASE__							:List[int]		   =    []
  SCREAMING_SNAKE_CASE__							:List[int]		   =    []

  def __init__(						self      :				str    ,      __a      :				int=None    ,      __a      :				List[str]=None    ,      __a      :				Optional[Any]=None    ,      __a      :				List[str]=None    ,      __a      :				Tuple="</s>"    ,      __a      :				Union[str, Any]="</s>"    ,      __a      :				str="<s>"    ,      __a      :				Any="<unk>"    ,      __a      :				List[str]="<pad>"    ,      __a      :				Optional[Any]="<mask>"    ,      **__a      :				Any    ,      )						-> Union[str, Any]:
        # Mask token behave like a normal word, i.e. include the space before it
        _UpperCamelCase					:		Dict  =				AddedToken(__a    ,      lstrip=__a    ,      rstrip=__a		) if isinstance(__a    ,      __a		) else mask_token

        _UpperCamelCase					:		str  =				kwargs.get("additional_special_tokens"    ,      []		)
        kwargs["additional_special_tokens"] += [
            code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            __a    ,      src_lang=__a    ,      tgt_lang=__a    ,      tokenizer_file=__a    ,      eos_token=__a    ,      sep_token=__a    ,      cls_token=__a    ,      unk_token=__a    ,      pad_token=__a    ,      mask_token=__a    ,      **__a    ,      )

        _UpperCamelCase					:		Optional[int]  =				vocab_file
        _UpperCamelCase					:		int  =				False if not self.vocab_file else True

        _UpperCamelCase					:		int  =				{
            lang_code: self.convert_tokens_to_ids(__a		) for lang_code in FAIRSEQ_LANGUAGE_CODES
        }

        _UpperCamelCase					:		List[str]  =				src_lang if src_lang is not None else "en_XX"
        _UpperCamelCase					:		Dict  =				tgt_lang
        _UpperCamelCase					:		int  =				self.lang_code_to_id[self._src_lang]
        self.set_src_lang_special_tokens(self._src_lang		)

  @property
  def       __SCREAMING_SNAKE_CASE (						self      :				Optional[int]		)						-> str:
        return self._src_lang

  @src_lang.setter
  def       __SCREAMING_SNAKE_CASE (						self      :				Union[str, Any]    ,      __a      :				str		)						-> None:
        _UpperCamelCase					:		int  =				new_src_lang
        self.set_src_lang_special_tokens(self._src_lang		)

  def       __SCREAMING_SNAKE_CASE (						self      :				Optional[int]    ,      __a      :				List[int]    ,      __a      :				Optional[List[int]] = None		)						-> List[int]:
        if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

  def       __SCREAMING_SNAKE_CASE (						self      :				Any    ,      __a      :				List[str]    ,      __a      :				str = "en_XX"    ,      __a      :				Optional[List[str]] = None    ,      __a      :				str = "ro_RO"    ,      **__a      :				Tuple    ,      )						-> BatchEncoding:
        _UpperCamelCase					:		List[Any]  =				src_lang
        _UpperCamelCase					:		Union[str, Any]  =				tgt_lang
        return super().prepare_seqaseq_batch(__a    ,      __a    ,      **__a		)

  def       __SCREAMING_SNAKE_CASE (						self      :				List[str]		)						-> int:
        return self.set_src_lang_special_tokens(self.src_lang		)

  def       __SCREAMING_SNAKE_CASE (						self      :				Tuple		)						-> Tuple:
        return self.set_tgt_lang_special_tokens(self.tgt_lang		)

  def       __SCREAMING_SNAKE_CASE (						self      :				List[str]    ,      __a      :				str		)						-> None:
        _UpperCamelCase					:		List[Any]  =				self.convert_tokens_to_ids(__a		)
        _UpperCamelCase					:		Union[str, Any]  =				[self.cur_lang_code_id]
        _UpperCamelCase					:		List[Any]  =				[self.eos_token_id]

        _UpperCamelCase					:		Union[str, Any]  =				self.convert_ids_to_tokens(self.prefix_tokens		)
        _UpperCamelCase					:		Union[str, Any]  =				self.convert_ids_to_tokens(self.suffix_tokens		)

        _UpperCamelCase					:		Optional[Any]  =				processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens		)		)    ,      )

  def       __SCREAMING_SNAKE_CASE (						self      :				Dict    ,      __a      :				str		)						-> None:
        _UpperCamelCase					:		Tuple  =				self.convert_tokens_to_ids(__a		)
        _UpperCamelCase					:		Union[str, Any]  =				[self.cur_lang_code_id]
        _UpperCamelCase					:		Optional[Any]  =				[self.eos_token_id]

        _UpperCamelCase					:		int  =				self.convert_ids_to_tokens(self.prefix_tokens		)
        _UpperCamelCase					:		Any  =				self.convert_ids_to_tokens(self.suffix_tokens		)

        _UpperCamelCase					:		Tuple  =				processors.TemplateProcessing(
            single=prefix_tokens_str + ["$A"] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens		)		)    ,      )

  def       __SCREAMING_SNAKE_CASE (						self      :				Dict    ,      __a      :				Optional[int]    ,      __a      :				str    ,      __a      :				Optional[str]    ,      __a      :				Optional[str]    ,      **__a      :				int		)						-> List[str]:
        if src_lang is None or tgt_lang is None:
              raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"		)
        _UpperCamelCase					:		List[str]  =				src_lang
        _UpperCamelCase					:		List[str]  =				self(__a    ,      add_special_tokens=__a    ,      return_tensors=__a    ,      **__a		)
        _UpperCamelCase					:		str  =				self.convert_tokens_to_ids(__a		)
        _UpperCamelCase					:		str  =				tgt_lang_id
        return inputs


  def       __SCREAMING_SNAKE_CASE (						self      :				Union[str, Any]    ,      __a      :				str    ,      __a      :				Optional[str] = None		)						-> Tuple[str]:
        if not self.can_save_slow_tokenizer:
              raise ValueError(
                  "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                  "tokenizer."		)

        if not os.path.isdir(__a		):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
              return
        _UpperCamelCase					:		int  =				os.path.join(
            __a    ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

        if os.path.abspath(self.vocab_file		) != os.path.abspath(__a		):
              copyfile(self.vocab_file    ,      __a		)

        return (out_vocab_file,)
