
from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


lowercase        =  logging.get_logger(__name__)


# General docstring
lowercase        =  '''MobileNetV1Config'''

# Base docstring
lowercase        =  '''google/mobilenet_v1_1.0_224'''
lowercase        =  [1, 1_0_2_4, 7, 7]

# Image classification docstring
lowercase        =  '''google/mobilenet_v1_1.0_224'''
lowercase        =  '''tabby, tabby cat'''


lowercase        =  [
    '''google/mobilenet_v1_1.0_224''',
    '''google/mobilenet_v1_0.75_192''',
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Dict			,				UpperCAmelCase__				:      List[str]			,				UpperCAmelCase__				:      str=None       )  ->    str:
							lowerCamelCase_							  =					{}

							if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					model.mobilenet_va
							else:
														lowerCamelCase_							  =					model

							lowerCamelCase_							  =					"""MobilenetV1/Conv2d_0/"""
							lowerCamelCase_							  =					backbone.conv_stem.convolution.weight
							lowerCamelCase_							  =					backbone.conv_stem.normalization.bias
							lowerCamelCase_							  =					backbone.conv_stem.normalization.weight
							lowerCamelCase_							  =					backbone.conv_stem.normalization.running_mean
							lowerCamelCase_							  =					backbone.conv_stem.normalization.running_var

							for i in range(1_3       ):
														lowerCamelCase_							  =					i + 1
														lowerCamelCase_							  =					i * 2

														lowerCamelCase_							  =					backbone.layer[pt_index]
														lowerCamelCase_							  =					F'''MobilenetV1/Conv2d_{tf_index}_depthwise/'''
														lowerCamelCase_							  =					pointer.convolution.weight
														lowerCamelCase_							  =					pointer.normalization.bias
														lowerCamelCase_							  =					pointer.normalization.weight
														lowerCamelCase_							  =					pointer.normalization.running_mean
														lowerCamelCase_							  =					pointer.normalization.running_var

														lowerCamelCase_							  =					backbone.layer[pt_index + 1]
														lowerCamelCase_							  =					F'''MobilenetV1/Conv2d_{tf_index}_pointwise/'''
														lowerCamelCase_							  =					pointer.convolution.weight
														lowerCamelCase_							  =					pointer.normalization.bias
														lowerCamelCase_							  =					pointer.normalization.weight
														lowerCamelCase_							  =					pointer.normalization.running_mean
														lowerCamelCase_							  =					pointer.normalization.running_var

							if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					"""MobilenetV1/Logits/Conv2d_1c_1x1/"""
														lowerCamelCase_							  =					model.classifier.weight
														lowerCamelCase_							  =					model.classifier.bias

							return tf_to_pt_map



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Dict			,				UpperCAmelCase__				:      Union[str, Any]			,				UpperCAmelCase__				:      Any       )  ->    List[str]:
							try:
														import numpy as np
														import tensorflow as tf
							except ImportError:
														logger.error(
														    """Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see """
														    """https://www.tensorflow.org/install/ for installation instructions."""       )
														raise

							# Load weights from TF model
							lowerCamelCase_							  =					tf.train.list_variables(UpperCAmelCase__       )
							lowerCamelCase_							  =					{}
							for name, shape in init_vars:
														logger.info(F'''Loading TF weight {name} with shape {shape}'''       )
														lowerCamelCase_							  =					tf.train.load_variable(UpperCAmelCase__			,				UpperCAmelCase__       )
														lowerCamelCase_							  =					array

							# Build TF to PyTorch weights loading map
							lowerCamelCase_							  =					_build_tf_to_pytorch_map(UpperCAmelCase__			,				UpperCAmelCase__			,				UpperCAmelCase__       )

							for name, pointer in tf_to_pt_map.items():
														logger.info(F'''Importing {name}'''       )
														if name not in tf_weights:
																					logger.info(F'''{name} not in tf pre-trained weights, skipping'''       )
																					continue

														lowerCamelCase_							  =					tf_weights[name]

														if "depthwise_weights" in name:
																					logger.info("""Transposing depthwise"""       )
																					lowerCamelCase_							  =					np.transpose(UpperCAmelCase__			,				(2, 3, 0, 1)       )
														elif "weights" in name:
																					logger.info("""Transposing"""       )
																					if len(pointer.shape       ) == 2:  # copying into linear layer
																												lowerCamelCase_							  =					array.squeeze().transpose()
																					else:
																												lowerCamelCase_							  =					np.transpose(UpperCAmelCase__			,				(3, 2, 0, 1)       )

														if pointer.shape != array.shape:
																					raise ValueError(F'''Pointer shape {pointer.shape} and array shape {array.shape} mismatched'''       )

														logger.info(F'''Initialize PyTorch weight {name} {array.shape}'''       )
														lowerCamelCase_							  =					torch.from_numpy(UpperCAmelCase__       )

														tf_weights.pop(UpperCAmelCase__			,				UpperCAmelCase__       )
														tf_weights.pop(name + """/RMSProp"""			,				UpperCAmelCase__       )
														tf_weights.pop(name + """/RMSProp_1"""			,				UpperCAmelCase__       )
														tf_weights.pop(name + """/ExponentialMovingAverage"""			,				UpperCAmelCase__       )

							logger.info(F'''Weights not copied to PyTorch model: {', '.join(tf_weights.keys()       )}'''       )
							return model



def      __lowerCAmelCase						(							UpperCAmelCase__				:      torch.Tensor			,				UpperCAmelCase__				:      nn.Convad       )  ->    torch.Tensor:
							lowerCamelCase_  ,   lowerCamelCase_							  =					features.shape[-2:]
							lowerCamelCase_  ,   lowerCamelCase_							  =					conv_layer.stride
							lowerCamelCase_  ,   lowerCamelCase_							  =					conv_layer.kernel_size

							if in_height % stride_height == 0:
														lowerCamelCase_							  =					max(kernel_height - stride_height			,				0       )
							else:
														lowerCamelCase_							  =					max(kernel_height - (in_height % stride_height)			,				0       )

							if in_width % stride_width == 0:
														lowerCamelCase_							  =					max(kernel_width - stride_width			,				0       )
							else:
														lowerCamelCase_							  =					max(kernel_width - (in_width % stride_width)			,				0       )

							lowerCamelCase_							  =					pad_along_width // 2
							lowerCamelCase_							  =					pad_along_width - pad_left
							lowerCamelCase_							  =					pad_along_height // 2
							lowerCamelCase_							  =					pad_along_height - pad_top

							lowerCamelCase_							  =					(pad_left, pad_right, pad_top, pad_bottom)
							return nn.functional.pad(UpperCAmelCase__			,				UpperCAmelCase__			,				"""constant"""			,				0.0       )





class       __A(				nn.Module       ):

					def __init__(	self					:     str   ,					__UpperCamelCase					:     MobileNetVaConfig   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     Optional[int] = 1   ,					__UpperCamelCase					:     Optional[int] = 1   ,					__UpperCamelCase					:     bool = False   ,					__UpperCamelCase					:     Optional[bool] = True   ,					__UpperCamelCase					:     Optional[bool or str] = True   ,					):
												super().__init__()
												lowerCamelCase_							  =					config

												if in_channels % groups != 0:
																			raise ValueError(F'''Input channels ({in_channels}) are not divisible by {groups} groups.'''   )
												if out_channels % groups != 0:
																			raise ValueError(F'''Output channels ({out_channels}) are not divisible by {groups} groups.'''   )

												lowerCamelCase_							  =					0 if config.tf_padding else int((kernel_size - 1) / 2   )

												lowerCamelCase_							  =					nn.Convad(
												    in_channels=__UpperCamelCase   ,					out_channels=__UpperCamelCase   ,					kernel_size=__UpperCamelCase   ,					stride=__UpperCamelCase   ,					padding=__UpperCamelCase   ,					groups=__UpperCamelCase   ,					bias=__UpperCamelCase   ,					padding_mode="""zeros"""   ,					)

												if use_normalization:
																			lowerCamelCase_							  =					nn.BatchNormad(
																			    num_features=__UpperCamelCase   ,					eps=config.layer_norm_eps   ,					momentum=0.9997   ,					affine=__UpperCamelCase   ,					track_running_stats=__UpperCamelCase   ,					)
												else:
																			lowerCamelCase_							  =					None

												if use_activation:
																			if isinstance(__UpperCamelCase   ,					__UpperCamelCase   ):
																										lowerCamelCase_							  =					ACTaFN[use_activation]
																			elif isinstance(config.hidden_act   ,					__UpperCamelCase   ):
																										lowerCamelCase_							  =					ACTaFN[config.hidden_act]
																			else:
																										lowerCamelCase_							  =					config.hidden_act
												else:
																			lowerCamelCase_							  =					None



					def    lowercase__      (	self					:     List[Any]   ,					__UpperCamelCase					:     torch.Tensor   ):
												if self.config.tf_padding:
																			lowerCamelCase_							  =					apply_tf_padding(__UpperCamelCase   ,					self.convolution   )
												lowerCamelCase_							  =					self.convolution(__UpperCamelCase   )
												if self.normalization is not None:
																			lowerCamelCase_							  =					self.normalization(__UpperCamelCase   )
												if self.activation is not None:
																			lowerCamelCase_							  =					self.activation(__UpperCamelCase   )
												return features





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    MobileNetVaConfig
					SCREAMING_SNAKE_CASE			    =    load_tf_weights_in_mobilenet_va
					SCREAMING_SNAKE_CASE			    =    '''mobilenet_v1'''
					SCREAMING_SNAKE_CASE			    =    '''pixel_values'''
					SCREAMING_SNAKE_CASE			    =    False

					def    lowercase__      (	self					:     Optional[Any]   ,					__UpperCamelCase					:     Union[nn.Linear, nn.Convad]   ):
												if isinstance(__UpperCamelCase   ,					(nn.Linear, nn.Convad)   ):
																			module.weight.data.normal_(mean=0.0   ,					std=self.config.initializer_range   )
																			if module.bias is not None:
																										module.bias.data.zero_()
												elif isinstance(__UpperCamelCase   ,					nn.BatchNormad   ):
																			module.bias.data.zero_()
																			module.weight.data.fill_(1.0   )
lowercase        =  r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

lowercase        =  r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`MobileNetV1ImageProcessor.__call__`] for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
'''





@add_start_docstrings(
    '''The bare MobileNetV1 model outputting raw hidden-states without any specific head on top.'''     ,		UpperCAmelCase     ,		)
class       __A(				UpperCAmelCase       ):

					def __init__(	self					:     str   ,					__UpperCamelCase					:     MobileNetVaConfig   ,					__UpperCamelCase					:     bool = True   ):
												super().__init__(__UpperCamelCase   )
												lowerCamelCase_							  =					config

												lowerCamelCase_							  =					3_2
												lowerCamelCase_							  =					max(int(depth * config.depth_multiplier   )   ,					config.min_depth   )

												lowerCamelCase_							  =					MobileNetVaConvLayer(
												    __UpperCamelCase   ,					in_channels=config.num_channels   ,					out_channels=__UpperCamelCase   ,					kernel_size=3   ,					stride=2   ,					)

												lowerCamelCase_							  =					[1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

												lowerCamelCase_							  =					nn.ModuleList()
												for i in range(1_3   ):
																			lowerCamelCase_							  =					out_channels

																			if strides[i] == 2 or i == 0:
																										depth *= 2
																										lowerCamelCase_							  =					max(int(depth * config.depth_multiplier   )   ,					config.min_depth   )

																			self.layer.append(
																			    MobileNetVaConvLayer(
																			        __UpperCamelCase   ,					in_channels=__UpperCamelCase   ,					out_channels=__UpperCamelCase   ,					kernel_size=3   ,					stride=strides[i]   ,					groups=__UpperCamelCase   ,					)   )

																			self.layer.append(
																			    MobileNetVaConvLayer(
																			        __UpperCamelCase   ,					in_channels=__UpperCamelCase   ,					out_channels=__UpperCamelCase   ,					kernel_size=1   ,					)   )

												lowerCamelCase_							  =					nn.AdaptiveAvgPoolad((1, 1)   ) if add_pooling_layer else None

												# Initialize weights and apply final processing
												self.post_init()

					def    lowercase__      (	self					:     Dict   ,					__UpperCamelCase					:     Tuple   ):
												raise NotImplementedError



					@add_start_docstrings_to_model_forward(__UpperCamelCase   )
					@add_code_sample_docstrings(
					    checkpoint=_CHECKPOINT_FOR_DOC   ,					output_type=__UpperCamelCase   ,					config_class=_CONFIG_FOR_DOC   ,					modality="""vision"""   ,					expected_output=_EXPECTED_OUTPUT_SHAPE   ,					)
					def    lowercase__      (	self					:     Optional[Any]   ,					__UpperCamelCase					:     Optional[torch.Tensor] = None   ,					__UpperCamelCase					:     Optional[bool] = None   ,					__UpperCamelCase					:     Optional[bool] = None   ,					):
												lowerCamelCase_							  =					(
												    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
												)
												lowerCamelCase_							  =					return_dict if return_dict is not None else self.config.use_return_dict

												if pixel_values is None:
																			raise ValueError("""You have to specify pixel_values"""   )

												lowerCamelCase_							  =					self.conv_stem(__UpperCamelCase   )

												lowerCamelCase_							  =					() if output_hidden_states else None

												for i, layer_module in enumerate(self.layer   ):
																			lowerCamelCase_							  =					layer_module(__UpperCamelCase   )

																			if output_hidden_states:
																										lowerCamelCase_							  =					all_hidden_states + (hidden_states,)

												lowerCamelCase_							  =					hidden_states

												if self.pooler is not None:
																			lowerCamelCase_							  =					torch.flatten(self.pooler(__UpperCamelCase   )   ,					start_dim=1   )
												else:
																			lowerCamelCase_							  =					None

												if not return_dict:
																			return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None   )

												return BaseModelOutputWithPoolingAndNoAttention(
												    last_hidden_state=__UpperCamelCase   ,					pooler_output=__UpperCamelCase   ,					hidden_states=__UpperCamelCase   ,					)





@add_start_docstrings(
    '''
    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    '''     ,		UpperCAmelCase     ,		)
class       __A(				UpperCAmelCase       ):

					def __init__(	self					:     Tuple   ,					__UpperCamelCase					:     MobileNetVaConfig   ):
												super().__init__(__UpperCamelCase   )

												lowerCamelCase_							  =					config.num_labels
												lowerCamelCase_							  =					MobileNetVaModel(__UpperCamelCase   )

												lowerCamelCase_							  =					self.mobilenet_va.layer[-1].convolution.out_channels

												# Classifier head
												lowerCamelCase_							  =					nn.Dropout(config.classifier_dropout_prob   ,					inplace=__UpperCamelCase   )
												lowerCamelCase_							  =					nn.Linear(__UpperCamelCase   ,					config.num_labels   ) if config.num_labels > 0 else nn.Identity()

												# Initialize weights and apply final processing
												self.post_init()



					@add_start_docstrings_to_model_forward(__UpperCamelCase   )
					@add_code_sample_docstrings(
					    checkpoint=_IMAGE_CLASS_CHECKPOINT   ,					output_type=__UpperCamelCase   ,					config_class=_CONFIG_FOR_DOC   ,					expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT   ,					)
					def    lowercase__      (	self					:     Union[str, Any]   ,					__UpperCamelCase					:     Optional[torch.Tensor] = None   ,					__UpperCamelCase					:     Optional[bool] = None   ,					__UpperCamelCase					:     Optional[torch.Tensor] = None   ,					__UpperCamelCase					:     Optional[bool] = None   ,					):
												lowerCamelCase_							  =					return_dict if return_dict is not None else self.config.use_return_dict

												lowerCamelCase_							  =					self.mobilenet_va(__UpperCamelCase   ,					output_hidden_states=__UpperCamelCase   ,					return_dict=__UpperCamelCase   )

												lowerCamelCase_							  =					outputs.pooler_output if return_dict else outputs[1]

												lowerCamelCase_							  =					self.classifier(self.dropout(__UpperCamelCase   )   )

												lowerCamelCase_							  =					None
												if labels is not None:
																			if self.config.problem_type is None:
																										if self.num_labels == 1:
																																	lowerCamelCase_							  =					"""regression"""
																										elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																																	lowerCamelCase_							  =					"""single_label_classification"""
																										else:
																																	lowerCamelCase_							  =					"""multi_label_classification"""

																			if self.config.problem_type == "regression":
																										lowerCamelCase_							  =					MSELoss()
																										if self.num_labels == 1:
																																	lowerCamelCase_							  =					loss_fct(logits.squeeze()   ,					labels.squeeze()   )
																										else:
																																	lowerCamelCase_							  =					loss_fct(__UpperCamelCase   ,					__UpperCamelCase   )
																			elif self.config.problem_type == "single_label_classification":
																										lowerCamelCase_							  =					CrossEntropyLoss()
																										lowerCamelCase_							  =					loss_fct(logits.view(-1   ,					self.num_labels   )   ,					labels.view(-1   )   )
																			elif self.config.problem_type == "multi_label_classification":
																										lowerCamelCase_							  =					BCEWithLogitsLoss()
																										lowerCamelCase_							  =					loss_fct(__UpperCamelCase   ,					__UpperCamelCase   )

												if not return_dict:
																			lowerCamelCase_							  =					(logits,) + outputs[2:]
																			return ((loss,) + output) if loss is not None else output

												return ImageClassifierOutputWithNoAttention(
												    loss=__UpperCamelCase   ,					logits=__UpperCamelCase   ,					hidden_states=outputs.hidden_states   ,					)







import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase        =  get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase        =  2_5_0_0_0_4
lowercase        =  2_5_0_0_2_0





@require_sentencepiece
@require_tokenizers
class       __A(				UpperCAmelCase     ,		unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizer
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizerFast
					SCREAMING_SNAKE_CASE			    =    True
					SCREAMING_SNAKE_CASE			    =    True

					def    lowercase__      (	self					:     Tuple   ):
												super().setUp()

												# We have a SentencePiece fixture for testing
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )
												tokenizer.save_pretrained(self.tmpdirname   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					"""<s>"""
												lowerCamelCase_							  =					0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase   )   ,					__UpperCamelCase   )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					list(self.get_tokenizer().get_vocab().keys()   )

												self.assertEqual(vocab_keys[0]   ,					"""<s>"""   )
												self.assertEqual(vocab_keys[1]   ,					"""<pad>"""   )
												self.assertEqual(vocab_keys[-1]   ,					"""<mask>"""   )
												self.assertEqual(len(__UpperCamelCase   )   ,					1_0_5_4   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												self.assertEqual(self.get_tokenizer().vocab_size   ,					1_0_5_4   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )

												lowerCamelCase_							  =					tokenizer.tokenize("""This is a test"""   )
												self.assertListEqual(__UpperCamelCase   ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]   )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__UpperCamelCase   )   ,					[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]   ,					)

												lowerCamelCase_							  =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]   ,					)
												lowerCamelCase_							  =					tokenizer.convert_tokens_to_ids(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[
												        value + tokenizer.fairseq_offset
												        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
												    ]   ,					)

												lowerCamelCase_							  =					tokenizer.convert_ids_to_tokens(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]   ,					)

					@slow
					def    lowercase__      (	self					:     Optional[int]   ):
												# fmt: off
												lowerCamelCase_							  =					{"""input_ids""": [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=__UpperCamelCase   ,					model_name="""facebook/mbart-large-50"""   ,					revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""   ,					)



					def    lowercase__      (	self					:     Dict   ):
												if not self.test_slow_tokenizer:
																			# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																			return

												lowerCamelCase_							  =					(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																										lowerCamelCase_							  =					self.rust_tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )
																										lowerCamelCase_							  =					self.tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )

																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files + the tokenizer.json file for the fast one
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )
																										lowerCamelCase_							  =					tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f   )
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )
																																	# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																																	# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=True
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=False
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it saved the tokenizer.json file
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )





@require_torch
@require_sentencepiece
@require_tokenizers
class       __A(				unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    '''facebook/mbart-large-50-one-to-many-mmt'''
					SCREAMING_SNAKE_CASE			    =    [
					    ''' UN Chief Says There Is No Military Solution in Syria''',
					    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
					]
					SCREAMING_SNAKE_CASE			    =    [
					    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
					    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
					    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
					    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
					]
					SCREAMING_SNAKE_CASE			    =    [EN_CODE, 8_2_7_4, 1_2_7_8_7_3, 2_5_9_1_6, 7, 8_6_2_2, 2_0_7_1, 4_3_8, 6_7_4_8_5, 5_3, 1_8_7_8_9_5, 2_3, 5_1_7_1_2, 2]

					@classmethod
					def    lowercase__      (	cls					:     Optional[Any]   ):
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(
												    cls.checkpoint_name   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   )
												lowerCamelCase_							  =					1
												return cls

					def    lowercase__      (	self					:     Union[str, Any]   ):
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]   ,					2_5_0_0_0_1   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]   ,					2_5_0_0_0_4   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]   ,					2_5_0_0_2_0   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]   ,					2_5_0_0_3_8   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Tuple   ):
												self.assertIn(__UpperCamelCase   ,					self.tokenizer.all_special_ids   )
												lowerCamelCase_							  =					[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
												lowerCamelCase_							  =					self.tokenizer.decode(__UpperCamelCase   ,					skip_special_tokens=__UpperCamelCase   )
												lowerCamelCase_							  =					self.tokenizer.decode(generated_ids[1:]   ,					skip_special_tokens=__UpperCamelCase   )
												self.assertEqual(__UpperCamelCase   ,					__UpperCamelCase   )
												self.assertNotIn(self.tokenizer.eos_token   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												lowerCamelCase_							  =					["""this is gunna be a long sentence """ * 2_0]
												assert isinstance(src_text[0]   ,					__UpperCamelCase   )
												lowerCamelCase_							  =					1_0
												lowerCamelCase_							  =					self.tokenizer(__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					truncation=__UpperCamelCase   ).input_ids[0]
												self.assertEqual(ids[0]   ,					__UpperCamelCase   )
												self.assertEqual(ids[-1]   ,					2   )
												self.assertEqual(len(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]   )   ,					[2_5_0_0_5_3, 2_5_0_0_0_1]   )

					def    lowercase__      (	self					:     Dict   ):
												lowerCamelCase_							  =					tempfile.mkdtemp()
												lowerCamelCase_							  =					self.tokenizer.fairseq_tokens_to_ids
												self.tokenizer.save_pretrained(__UpperCamelCase   )
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(__UpperCamelCase   )
												self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,					__UpperCamelCase   )

					@require_torch
					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
												assert batch.input_ids[1][0] == EN_CODE
												assert batch.input_ids[1][-1] == 2
												assert batch.labels[1][0] == RO_CODE
												assert batch.labels[1][-1] == 2
												assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]

					@require_torch
					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					self.tokenizer(
												    self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=len(self.expected_src_tokens   )   ,					return_tensors="""pt"""   ,					)
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												self.assertIsInstance(__UpperCamelCase   ,					__UpperCamelCase   )

												self.assertEqual((2, 1_4)   ,					batch.input_ids.shape   )
												self.assertEqual((2, 1_4)   ,					batch.attention_mask.shape   )
												lowerCamelCase_							  =					batch.input_ids.tolist()[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )
												self.assertEqual(2   ,					batch.decoder_input_ids[0, 0]   )  # decoder_start_token_id
												# Test that special tokens are reset
												self.assertEqual(self.tokenizer.prefix_tokens   ,					[EN_CODE]   )
												self.assertEqual(self.tokenizer.suffix_tokens   ,					[self.tokenizer.eos_token_id]   )

					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=3   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					self.tokenizer(
												    text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=1_0   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					targets["""input_ids"""]
												lowerCamelCase_							  =					shift_tokens_right(__UpperCamelCase   ,					self.tokenizer.pad_token_id   )

												self.assertEqual(batch.input_ids.shape[1]   ,					3   )
												self.assertEqual(batch.decoder_input_ids.shape[1]   ,					1_0   )



					@require_torch
					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer._build_translation_inputs(
												    """A test"""   ,					return_tensors="""pt"""   ,					src_lang="""en_XX"""   ,					tgt_lang="""ar_AR"""   )

												self.assertEqual(
												    nested_simplify(__UpperCamelCase   )   ,					{
												        # en_XX, A, test, EOS
												        """input_ids""": [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
												        """attention_mask""": [[1, 1, 1, 1]],
												        # ar_AR
												        """forced_bos_token_id""": 2_5_0_0_0_1,
												    }   ,					)





