






from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


UpperCAmelCase__							  =						logging.get_logger(__name__)


# General docstring
UpperCAmelCase__							  =						"MobileNetV1Config"

# Base docstring
UpperCAmelCase__							  =						"google/mobilenet_v1_1.0_224"
UpperCAmelCase__							  =						[1, 1024, 7, 7]

# Image classification docstring
UpperCAmelCase__							  =						"google/mobilenet_v1_1.0_224"
UpperCAmelCase__							  =						"tabby, tabby cat"


UpperCAmelCase__							  =						[
    "google/mobilenet_v1_1.0_224",
    "google/mobilenet_v1_0.75_192",
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]
def 		_a       ( a					:List[str]      ,				a					:Any      ,				a					:Optional[Any]=None					)				->     Any:
 a						    = {}

 if isinstance(a      ,				a					):
  a						    = model.mobilenet_va
 else:
  a						    = model

 a						    = '''MobilenetV1/Conv2d_0/'''
 a						    = backbone.conv_stem.convolution.weight
 a						    = backbone.conv_stem.normalization.bias
 a						    = backbone.conv_stem.normalization.weight
 a						    = backbone.conv_stem.normalization.running_mean
 a						    = backbone.conv_stem.normalization.running_var

 for i in range(13					):
  a						    = i + 1
  a						    = i * 2

  a						    = backbone.layer[pt_index]
  a						    = F"""MobilenetV1/Conv2d_{tf_index}_depthwise/"""
  a						    = pointer.convolution.weight
  a						    = pointer.normalization.bias
  a						    = pointer.normalization.weight
  a						    = pointer.normalization.running_mean
  a						    = pointer.normalization.running_var

  a						    = backbone.layer[pt_index + 1]
  a						    = F"""MobilenetV1/Conv2d_{tf_index}_pointwise/"""
  a						    = pointer.convolution.weight
  a						    = pointer.normalization.bias
  a						    = pointer.normalization.weight
  a						    = pointer.normalization.running_mean
  a						    = pointer.normalization.running_var

 if isinstance(a      ,				a					):
  a						    = '''MobilenetV1/Logits/Conv2d_1c_1x1/'''
  a						    = model.classifier.weight
  a						    = model.classifier.bias

 return tf_to_pt_map
def 		_a       ( a					:int      ,				a					:Tuple      ,				a					:Any					)				->     Optional[int]:
 try:
  import numpy as np
  import tensorflow as tf
 except ImportError:
  logger.error(
      '''Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see '''
      '''https://www.tensorflow.org/install/ for installation instructions.'''					)
  raise

 # Load weights from TF model
 a						    = tf.train.list_variables(a					)
 a						    = {}
 for name, shape in init_vars:
  logger.info(F"""Loading TF weight {name} with shape {shape}"""					)
  a						    = tf.train.load_variable(a      ,				a					)
  a						    = array

 # Build TF to PyTorch weights loading map
 a						    = _build_tf_to_pytorch_map(a      ,				a      ,				a					)

 for name, pointer in tf_to_pt_map.items():
  logger.info(F"""Importing {name}"""					)
  if name not in tf_weights:
   logger.info(F"""{name} not in tf pre-trained weights, skipping"""					)
   continue

  a						    = tf_weights[name]

  if "depthwise_weights" in name:
   logger.info('''Transposing depthwise'''					)
   a						    = np.transpose(a      ,				(2, 3, 0, 1)					)
  elif "weights" in name:
   logger.info('''Transposing'''					)
   if len(pointer.shape					) == 2:  # copying into linear layer
    a						    = array.squeeze().transpose()
   else:
    a						    = np.transpose(a      ,				(3, 2, 0, 1)					)

  if pointer.shape != array.shape:
   raise ValueError(F"""Pointer shape {pointer.shape} and array shape {array.shape} mismatched"""					)

  logger.info(F"""Initialize PyTorch weight {name} {array.shape}"""					)
  a						    = torch.from_numpy(a					)

  tf_weights.pop(a      ,				a					)
  tf_weights.pop(name + '''/RMSProp'''      ,				a					)
  tf_weights.pop(name + '''/RMSProp_1'''      ,				a					)
  tf_weights.pop(name + '''/ExponentialMovingAverage'''      ,				a					)

 logger.info(F"""Weights not copied to PyTorch model: {', '.join(tf_weights.keys()					)}"""					)
 return model
def 		_a       ( a					:torch.Tensor      ,				a					:nn.Convad					)				->     torch.Tensor:
 a    ,	a						    = features.shape[-2:]
 a    ,	a						    = conv_layer.stride
 a    ,	a						    = conv_layer.kernel_size

 if in_height % stride_height == 0:
  a						    = max(kernel_height - stride_height      ,				0					)
 else:
  a						    = max(kernel_height - (in_height % stride_height)      ,				0					)

 if in_width % stride_width == 0:
  a						    = max(kernel_width - stride_width      ,				0					)
 else:
  a						    = max(kernel_width - (in_width % stride_width)      ,				0					)

 a						    = pad_along_width // 2
 a						    = pad_along_width - pad_left
 a						    = pad_along_height // 2
 a						    = pad_along_height - pad_top

 a						    = (pad_left, pad_right, pad_top, pad_bottom)
 return nn.functional.pad(a      ,				a      ,				'''constant'''      ,				0.0					)






class 						lowercase_   (      nn.Module  ):






      '''simple docstring'''




      def __init__( self :		Optional[int]					,   __UpperCAmelCase :		MobileNetVaConfig					,   __UpperCAmelCase :		int					,   __UpperCAmelCase :		int					,   __UpperCAmelCase :		int					,   __UpperCAmelCase :		Optional[int] = 1					,   __UpperCAmelCase :		Optional[int] = 1					,   __UpperCAmelCase :		bool = False					,   __UpperCAmelCase :		Optional[bool] = True					,   __UpperCAmelCase :		Optional[bool or str] = True					,   )       ->None:




       """simple docstring"""


       super().__init__()
       a						    = config

       if in_channels % groups != 0:
        raise ValueError(F"""Input channels ({in_channels}) are not divisible by {groups} groups."""    )
       if out_channels % groups != 0:
        raise ValueError(F"""Output channels ({out_channels}) are not divisible by {groups} groups."""    )

       a						    = 0 if config.tf_padding else int((kernel_size - 1) / 2    )

       a						    = nn.Convad(
           in_channels=__UpperCAmelCase					,   out_channels=__UpperCAmelCase					,   kernel_size=__UpperCAmelCase					,   stride=__UpperCAmelCase					,   padding=__UpperCAmelCase					,   groups=__UpperCAmelCase					,   bias=__UpperCAmelCase					,   padding_mode='''zeros'''					,   )

       if use_normalization:
        a						    = nn.BatchNormad(
            num_features=__UpperCAmelCase					,   eps=config.layer_norm_eps					,   momentum=0.9997					,   affine=__UpperCAmelCase					,   track_running_stats=__UpperCAmelCase					,   )
       else:
        a						    = None

       if use_activation:
        if isinstance(__UpperCAmelCase					,   __UpperCAmelCase    ):
         a						    = ACTaFN[use_activation]
        elif isinstance(config.hidden_act					,   __UpperCAmelCase    ):
         a						    = ACTaFN[config.hidden_act]
        else:
         a						    = config.hidden_act
       else:
        a						    = None







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		torch.Tensor    )       ->torch.Tensor:




       """simple docstring"""


       if self.config.tf_padding:
        a						    = apply_tf_padding(__UpperCAmelCase					,   self.convolution    )
       a						    = self.convolution(__UpperCAmelCase    )
       if self.normalization is not None:
        a						    = self.normalization(__UpperCAmelCase    )
       if self.activation is not None:
        a						    = self.activation(__UpperCAmelCase    )
       return features






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    MobileNetVaConfig
      __snake_case	    =    load_tf_weights_in_mobilenet_va
      __snake_case	    =    '''mobilenet_v1'''
      __snake_case	    =    '''pixel_values'''
      __snake_case	    =    False







      def  __lowerCAmelCase   ( self :		Optional[int]					,   __UpperCAmelCase :		Union[nn.Linear, nn.Convad]    )       ->None:




       """simple docstring"""


       if isinstance(__UpperCAmelCase					,   (nn.Linear, nn.Convad)    ):
        module.weight.data.normal_(mean=0.0					,   std=self.config.initializer_range    )
        if module.bias is not None:
         module.bias.data.zero_()
       elif isinstance(__UpperCAmelCase					,   nn.BatchNormad    ):
        module.bias.data.zero_()
        module.weight.data.fill_(1.0    )
UpperCAmelCase__							  =						R"\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

UpperCAmelCase__							  =						R"\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`MobileNetV1ImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n"






@add_start_docstrings(
    '''The bare MobileNetV1 model outputting raw hidden-states without any specific head on top.'''					,    lowercase					,    )
class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      def __init__( self :		Tuple					,   __UpperCAmelCase :		MobileNetVaConfig					,   __UpperCAmelCase :		bool = True    )       ->str:




       """simple docstring"""


       super().__init__(__UpperCAmelCase    )
       a						    = config

       a						    = 32
       a						    = max(int(depth * config.depth_multiplier    )					,   config.min_depth    )

       a						    = MobileNetVaConvLayer(
           __UpperCAmelCase					,   in_channels=config.num_channels					,   out_channels=__UpperCAmelCase					,   kernel_size=3					,   stride=2					,   )

       a						    = [1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

       a						    = nn.ModuleList()
       for i in range(13    ):
        a						    = out_channels

        if strides[i] == 2 or i == 0:
         depth *= 2
         a						    = max(int(depth * config.depth_multiplier    )					,   config.min_depth    )

        self.layer.append(
            MobileNetVaConvLayer(
                __UpperCAmelCase					,   in_channels=__UpperCAmelCase					,   out_channels=__UpperCAmelCase					,   kernel_size=3					,   stride=strides[i]					,   groups=__UpperCAmelCase					,   )    )

        self.layer.append(
            MobileNetVaConvLayer(
                __UpperCAmelCase					,   in_channels=__UpperCAmelCase					,   out_channels=__UpperCAmelCase					,   kernel_size=1					,   )    )

       a						    = nn.AdaptiveAvgPoolad((1, 1)    ) if add_pooling_layer else None

       # Initialize weights and apply final processing
       self.post_init()







      def  __lowerCAmelCase   ( self :		List[Any]					,   __UpperCAmelCase :		List[str]    )       ->List[str]:




       """simple docstring"""


       raise NotImplementedError







      @add_start_docstrings_to_model_forward(__UpperCAmelCase    )
      @add_code_sample_docstrings(
          checkpoint=_CHECKPOINT_FOR_DOC					,   output_type=__UpperCAmelCase					,   config_class=_CONFIG_FOR_DOC					,   modality='''vision'''					,   expected_output=_EXPECTED_OUTPUT_SHAPE					,   )
      def  __lowerCAmelCase   ( self :		Tuple					,   __UpperCAmelCase :		Optional[torch.Tensor] = None					,   __UpperCAmelCase :		Optional[bool] = None					,   __UpperCAmelCase :		Optional[bool] = None					,   )       ->Union[tuple, BaseModelOutputWithPoolingAndNoAttention]:




       """simple docstring"""


       a						    = (
           output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
       )
       a						    = return_dict if return_dict is not None else self.config.use_return_dict

       if pixel_values is None:
        raise ValueError('''You have to specify pixel_values'''    )

       a						    = self.conv_stem(__UpperCAmelCase    )

       a						    = () if output_hidden_states else None

       for i, layer_module in enumerate(self.layer    ):
        a						    = layer_module(__UpperCAmelCase    )

        if output_hidden_states:
         a						    = all_hidden_states + (hidden_states,)

       a						    = hidden_states

       if self.pooler is not None:
        a						    = torch.flatten(self.pooler(__UpperCAmelCase    )					,   start_dim=1    )
       else:
        a						    = None

       if not return_dict:
        return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None    )

       return BaseModelOutputWithPoolingAndNoAttention(
           last_hidden_state=__UpperCAmelCase					,   pooler_output=__UpperCAmelCase					,   hidden_states=__UpperCAmelCase					,   )






@add_start_docstrings(
    '''
    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    '''					,    lowercase					,    )
class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      def __init__( self :		Tuple					,   __UpperCAmelCase :		MobileNetVaConfig    )       ->None:




       """simple docstring"""


       super().__init__(__UpperCAmelCase    )

       a						    = config.num_labels
       a						    = MobileNetVaModel(__UpperCAmelCase    )

       a						    = self.mobilenet_va.layer[-1].convolution.out_channels

       # Classifier head
       a						    = nn.Dropout(config.classifier_dropout_prob					,   inplace=__UpperCAmelCase    )
       a						    = nn.Linear(__UpperCAmelCase					,   config.num_labels    ) if config.num_labels > 0 else nn.Identity()

       # Initialize weights and apply final processing
       self.post_init()







      @add_start_docstrings_to_model_forward(__UpperCAmelCase    )
      @add_code_sample_docstrings(
          checkpoint=_IMAGE_CLASS_CHECKPOINT					,   output_type=__UpperCAmelCase					,   config_class=_CONFIG_FOR_DOC					,   expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT					,   )
      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		Optional[torch.Tensor] = None					,   __UpperCAmelCase :		Optional[bool] = None					,   __UpperCAmelCase :		Optional[torch.Tensor] = None					,   __UpperCAmelCase :		Optional[bool] = None					,   )       ->Union[tuple, ImageClassifierOutputWithNoAttention]:




       """simple docstring"""


       a						    = return_dict if return_dict is not None else self.config.use_return_dict

       a						    = self.mobilenet_va(__UpperCAmelCase					,   output_hidden_states=__UpperCAmelCase					,   return_dict=__UpperCAmelCase    )

       a						    = outputs.pooler_output if return_dict else outputs[1]

       a						    = self.classifier(self.dropout(__UpperCAmelCase    )    )

       a						    = None
       if labels is not None:
        if self.config.problem_type is None:
         if self.num_labels == 1:
          a						    = '''regression'''
         elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
          a						    = '''single_label_classification'''
         else:
          a						    = '''multi_label_classification'''

        if self.config.problem_type == "regression":
         a						    = MSELoss()
         if self.num_labels == 1:
          a						    = loss_fct(logits.squeeze()					,   labels.squeeze()    )
         else:
          a						    = loss_fct(__UpperCAmelCase					,   __UpperCAmelCase    )
        elif self.config.problem_type == "single_label_classification":
         a						    = CrossEntropyLoss()
         a						    = loss_fct(logits.view(-1					,   self.num_labels    )					,   labels.view(-1    )    )
        elif self.config.problem_type == "multi_label_classification":
         a						    = BCEWithLogitsLoss()
         a						    = loss_fct(__UpperCAmelCase					,   __UpperCAmelCase    )

       if not return_dict:
        a						    = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

       return ImageClassifierOutputWithNoAttention(
           loss=__UpperCAmelCase					,   logits=__UpperCAmelCase					,   hidden_states=outputs.hidden_states					,   )










import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__							  =						logging.get_logger(__name__)

UpperCAmelCase__							  =						"▁"

UpperCAmelCase__							  =						{"vocab_file": "sentencepiece.bpe.model"}

UpperCAmelCase__							  =						{
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model"
        ),
    }
}

UpperCAmelCase__							  =						{
    "facebook/nllb-200-distilled-600M": 1024,
}

# fmt: off
UpperCAmelCase__							  =						["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    VOCAB_FILES_NAMES
      __snake_case	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case	    =    PRETRAINED_VOCAB_FILES_MAP
      __snake_case	    =    ['''input_ids''', '''attention_mask''']

      __snake_case	    =    []
      __snake_case	    =    []







      def __init__( self :		int					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[int]="<s>"					,   __UpperCAmelCase :		Union[str, Any]="</s>"					,   __UpperCAmelCase :		List[Any]="</s>"					,   __UpperCAmelCase :		List[Any]="<s>"					,   __UpperCAmelCase :		Optional[int]="<unk>"					,   __UpperCAmelCase :		Optional[int]="<pad>"					,   __UpperCAmelCase :		Tuple="<mask>"					,   __UpperCAmelCase :		Any=None					,   __UpperCAmelCase :		Dict=None					,   __UpperCAmelCase :		int=None					,   __UpperCAmelCase :		Optional[Dict[str, Any]] = None					,   __UpperCAmelCase :		List[Any]=None					,   __UpperCAmelCase :		int=False					,   **__UpperCAmelCase :		Dict					,   )       ->List[str]:




       """simple docstring"""


       a						    = AddedToken(__UpperCAmelCase					,   lstrip=__UpperCAmelCase					,   rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase					,   __UpperCAmelCase    ) else mask_token

       a						    = {} if sp_model_kwargs is None else sp_model_kwargs
       a						    = legacy_behaviour

       super().__init__(
           bos_token=__UpperCAmelCase					,   eos_token=__UpperCAmelCase					,   unk_token=__UpperCAmelCase					,   sep_token=__UpperCAmelCase					,   cls_token=__UpperCAmelCase					,   pad_token=__UpperCAmelCase					,   mask_token=__UpperCAmelCase					,   tokenizer_file=__UpperCAmelCase					,   src_lang=__UpperCAmelCase					,   tgt_lang=__UpperCAmelCase					,   additional_special_tokens=__UpperCAmelCase					,   sp_model_kwargs=self.sp_model_kwargs					,   legacy_behaviour=__UpperCAmelCase					,   **__UpperCAmelCase					,   )

       a						    = spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(__UpperCAmelCase    )    )
       a						    = vocab_file
       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
       # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
       # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

       # Mimic fairseq token-to-id alignment for the first 4 token
       a						    = {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       a						    = 1

       a						    = len(self.sp_model    )
       a						    = {
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__UpperCAmelCase    )
       }
       a						    = {v: k for k, v in self.lang_code_to_id.items()}
       a						    = len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
       a						    = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
       a						    = list(self.lang_code_to_id.keys()    )

       if additional_special_tokens is not None:
        # Only add those special tokens if they are not already there.
        self._additional_special_tokens.extend(
            [t for t in additional_special_tokens if t not in self._additional_special_tokens]    )

       a						    = src_lang if src_lang is not None else '''eng_Latn'''
       a						    = self.lang_code_to_id[self._src_lang]
       a						    = tgt_lang
       self.set_src_lang_special_tokens(self._src_lang    )







      def __getstate__( self :		str    )       ->Any:




       """simple docstring"""


       a						    = self.__dict__.copy()
       a						    = None
       a						    = self.sp_model.serialized_model_proto()
       return state







      def __setstate__( self :		List[str]					,   __UpperCAmelCase :		Optional[Any]    )       ->int:




       """simple docstring"""


       a						    = d

       # for backward compatibility
       if not hasattr(self					,   '''sp_model_kwargs'''    ):
        a						    = {}

       a						    = spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )







      @property
      def  __lowerCAmelCase   ( self :		Tuple    )       ->Optional[Any]:




       """simple docstring"""


       return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token







      @property
      def  __lowerCAmelCase   ( self :		Any    )       ->str:




       """simple docstring"""


       return self._src_lang







      @src_lang.setter
      def  __lowerCAmelCase   ( self :		List[Any]					,   __UpperCAmelCase :		str    )       ->None:




       """simple docstring"""


       a						    = new_src_lang
       self.set_src_lang_special_tokens(self._src_lang    )







      def  __lowerCAmelCase   ( self :		Any					,   __UpperCAmelCase :		List[int]					,   __UpperCAmelCase :		Optional[List[int]] = None					,   __UpperCAmelCase :		bool = False    )       ->List[int]:




       """simple docstring"""


       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__UpperCAmelCase					,   token_ids_a=__UpperCAmelCase					,   already_has_special_tokens=__UpperCAmelCase    )

       a						    = [1] * len(self.prefix_tokens    )
       a						    = [1] * len(self.suffix_tokens    )
       if token_ids_a is None:
        return prefix_ones + ([0] * len(__UpperCAmelCase    )) + suffix_ones
       return prefix_ones + ([0] * len(__UpperCAmelCase    )) + ([0] * len(__UpperCAmelCase    )) + suffix_ones







      def  __lowerCAmelCase   ( self :		Any					,   __UpperCAmelCase :		List[int]					,   __UpperCAmelCase :		Optional[List[int]] = None    )       ->List[int]:




       """simple docstring"""


       if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







      def  __lowerCAmelCase   ( self :		int					,   __UpperCAmelCase :		List[int]					,   __UpperCAmelCase :		Optional[List[int]] = None    )       ->List[int]:




       """simple docstring"""


       a						    = [self.sep_token_id]
       a						    = [self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		Optional[Any]					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str]					,   __UpperCAmelCase :		Optional[str]					,   **__UpperCAmelCase :		Tuple    )       ->Tuple:




       """simple docstring"""


       if src_lang is None or tgt_lang is None:
        raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''    )
       a						    = src_lang
       a						    = self(__UpperCAmelCase					,   add_special_tokens=__UpperCAmelCase					,   return_tensors=__UpperCAmelCase					,   **__UpperCAmelCase    )
       a						    = self.convert_tokens_to_ids(__UpperCAmelCase    )
       a						    = tgt_lang_id
       return inputs







      def  __lowerCAmelCase   ( self :		Optional[int]    )       ->Union[str, Any]:




       """simple docstring"""


       a						    = {self.convert_ids_to_tokens(__UpperCAmelCase    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab







      def  __lowerCAmelCase   ( self :		List[Any]					,   __UpperCAmelCase :		str    )       ->List[str]:




       """simple docstring"""


       return self.sp_model.encode(__UpperCAmelCase					,   out_type=__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		Optional[int]					,   __UpperCAmelCase :		str    )       ->List[Any]:




       """simple docstring"""


       if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
       a						    = self.sp_model.PieceToId(__UpperCAmelCase    )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







      def  __lowerCAmelCase   ( self :		Union[str, Any]					,   __UpperCAmelCase :		int    )       ->Optional[Any]:




       """simple docstring"""


       if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )







      def  __lowerCAmelCase   ( self :		Tuple					,   __UpperCAmelCase :		List[str]    )       ->List[Any]:




       """simple docstring"""


       a						    = ''''''.join(__UpperCAmelCase    ).replace(__UpperCAmelCase					,   ''' '''    ).strip()
       return out_string







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str] = None    )       ->Tuple[str]:




       """simple docstring"""


       if not os.path.isdir(__UpperCAmelCase    ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
        return
       a						    = os.path.join(
           __UpperCAmelCase					,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(__UpperCAmelCase    ) and os.path.isfile(self.vocab_file    ):
        copyfile(self.vocab_file					,   __UpperCAmelCase    )
       elif not os.path.isfile(self.vocab_file    ):
        with open(__UpperCAmelCase					,   '''wb'''    ) as fi:
         a						    = self.sp_model.serialized_model_proto()
         fi.write(__UpperCAmelCase    )

       return (out_vocab_file,)







      def  __lowerCAmelCase   ( self :		List[Any]					,   __UpperCAmelCase :		List[str]					,   __UpperCAmelCase :		str = "eng_Latn"					,   __UpperCAmelCase :		Optional[List[str]] = None					,   __UpperCAmelCase :		str = "fra_Latn"					,   **__UpperCAmelCase :		Optional[int]					,   )       ->BatchEncoding:




       """simple docstring"""


       a						    = src_lang
       a						    = tgt_lang
       return super().prepare_seqaseq_batch(__UpperCAmelCase					,   __UpperCAmelCase					,   **__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		Dict    )       ->List[str]:




       """simple docstring"""


       return self.set_src_lang_special_tokens(self.src_lang    )







      def  __lowerCAmelCase   ( self :		Any    )       ->Any:




       """simple docstring"""


       return self.set_tgt_lang_special_tokens(self.tgt_lang    )







      def  __lowerCAmelCase   ( self :		Any					,   __UpperCAmelCase :		Tuple    )       ->None:




       """simple docstring"""


       a						    = self.lang_code_to_id[src_lang]
       if self.legacy_behaviour:
        a						    = []
        a						    = [self.eos_token_id, self.cur_lang_code]
       else:
        a						    = [self.cur_lang_code]
        a						    = [self.eos_token_id]







      def  __lowerCAmelCase   ( self :		List[str]					,   __UpperCAmelCase :		str    )       ->None:




       """simple docstring"""


       a						    = self.lang_code_to_id[lang]
       if self.legacy_behaviour:
        a						    = []
        a						    = [self.eos_token_id, self.cur_lang_code]
       else:
        a						    = [self.cur_lang_code]
        a						    = [self.eos_token_id]


