
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Dict			,				UpperCAmelCase__				:      Optional[int]			,				UpperCAmelCase__				:      List[Any]=0       )  ->    int:
							# Format the message.
							if name is None:
														lowerCamelCase_							  =					None
							else:
														lowerCamelCase_							  =					""".""" * max(0			,				spaces - 2       ) + """# {:""" + str(5_0 - spaces       ) + """s}"""
														lowerCamelCase_							  =					fmt.format(UpperCAmelCase__       )

							# Print and recurse (if needed).
							if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														if msg is not None:
																					print(UpperCAmelCase__       )
														for k in val.keys():
																					recursive_print(UpperCAmelCase__			,				val[k]			,				spaces + 2       )
							elif isinstance(UpperCAmelCase__			,				torch.Tensor       ):
														print(UpperCAmelCase__			,				""":"""			,				val.size()       )
							else:
														print(UpperCAmelCase__			,				""":"""			,				UpperCAmelCase__       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Optional[Any]			,				UpperCAmelCase__				:      Any			,				UpperCAmelCase__				:      List[str]			,				UpperCAmelCase__				:      Optional[int]			,				UpperCAmelCase__				:      str       )  ->    Dict:
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							lowerCamelCase_							  =					param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														lowerCamelCase_							  =					(num_heads, hidden_size, num_splits) + input_shape[1:]
														lowerCamelCase_							  =					param.view(*UpperCAmelCase__       )
														lowerCamelCase_							  =					param.transpose(0			,				2       )
														lowerCamelCase_							  =					param.transpose(1			,				2       ).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														lowerCamelCase_							  =					(num_heads, num_splits, hidden_size) + input_shape[1:]
														lowerCamelCase_							  =					param.view(*UpperCAmelCase__       )
														lowerCamelCase_							  =					param.transpose(0			,				1       ).contiguous()
							lowerCamelCase_							  =					param.view(*UpperCAmelCase__       )
							return param



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[str]			,				UpperCAmelCase__				:      Tuple			,				UpperCAmelCase__				:      List[str]       )  ->    int:
							# The converted output model.
							lowerCamelCase_							  =					{}

							# old versions did not store training args
							lowerCamelCase_							  =					input_state_dict.get("""args"""			,				UpperCAmelCase__       )
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														lowerCamelCase_							  =					ds_args.padded_vocab_size
														lowerCamelCase_							  =					ds_args.max_position_embeddings
														lowerCamelCase_							  =					ds_args.hidden_size
														lowerCamelCase_							  =					ds_args.num_layers
														lowerCamelCase_							  =					ds_args.num_attention_heads
														lowerCamelCase_							  =					ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							lowerCamelCase_							  =					config.n_head
							# The hidden_size per head.
							lowerCamelCase_							  =					config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														lowerCamelCase_							  =					input_state_dict["""checkpoint_version"""]
							else:
														lowerCamelCase_							  =					0.0

							# The model.
							lowerCamelCase_							  =					input_state_dict["""model"""]
							# The language model.
							lowerCamelCase_							  =					model["""language_model"""]
							# The embeddings.
							lowerCamelCase_							  =					lm["""embedding"""]

							# The word embeddings.
							lowerCamelCase_							  =					embeddings["""word_embeddings"""]["""weight"""]
							# Truncate the embedding table to vocab_size rows.
							lowerCamelCase_							  =					word_embeddings[: config.vocab_size, :]
							lowerCamelCase_							  =					word_embeddings

							# The position embeddings.
							lowerCamelCase_							  =					embeddings["""position_embeddings"""]["""weight"""]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							lowerCamelCase_							  =					pos_embeddings.size(0       )
							if n_positions != config.n_positions:
														raise ValueError(
														    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''       )
							# Store the position embeddings.
							lowerCamelCase_							  =					pos_embeddings

							# The transformer.
							lowerCamelCase_							  =					lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

							# The regex to extract layer names.
							lowerCamelCase_							  =					re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""       )

							# The simple map of names for "automated" rules.
							lowerCamelCase_							  =					{
							    """attention.dense""": """.attn.c_proj.""",
							    """self_attention.dense""": """.attn.c_proj.""",
							    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
							    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														lowerCamelCase_							  =					layer_re.match(UpperCAmelCase__       )

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														lowerCamelCase_							  =					int(m.group(1       )       )
														# The name of the operation.
														lowerCamelCase_							  =					m.group(2       )
														# Is it a weight or a bias?
														lowerCamelCase_							  =					m.group(3       )

														# The name of the layer.
														lowerCamelCase_							  =					F'''transformer.h.{layer_idx}'''

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("""layernorm"""       ):
																					lowerCamelCase_							  =					"""ln_1""" if op_name.startswith("""input"""       ) else """ln_2"""
																					lowerCamelCase_							  =					val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					lowerCamelCase_							  =					torch.tril(torch.ones((n_positions, n_positions)			,				dtype=torch.floataa       )       ).view(
																					    1			,				1			,				UpperCAmelCase__			,				UpperCAmelCase__       )
																					lowerCamelCase_							  =					causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					lowerCamelCase_							  =					torch.tensor(-1e4			,				dtype=torch.floataa       )
																					lowerCamelCase_							  =					masked_bias

																					lowerCamelCase_							  =					fix_query_key_value_ordering(UpperCAmelCase__			,				UpperCAmelCase__			,				3			,				UpperCAmelCase__			,				UpperCAmelCase__       )
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					lowerCamelCase_							  =					out_val.transpose(0			,				1       ).contiguous()
																					# Store.
																					lowerCamelCase_							  =					out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					lowerCamelCase_							  =					fix_query_key_value_ordering(UpperCAmelCase__			,				UpperCAmelCase__			,				3			,				UpperCAmelCase__			,				UpperCAmelCase__       )
																					# Store. No change of shape.
																					lowerCamelCase_							  =					out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					lowerCamelCase_							  =					megatron_to_transformers[op_name]
																					lowerCamelCase_							  =					val.transpose(0			,				1       )

														# Copy the bias.
														elif weight_or_bias == "bias":
																					lowerCamelCase_							  =					megatron_to_transformers[op_name]
																					lowerCamelCase_							  =					val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							lowerCamelCase_							  =					transformer["""final_layernorm.weight"""]
							lowerCamelCase_							  =					transformer["""final_layernorm.bias"""]

							# For LM head, transformers' wants the matrix to weight embeddings.
							lowerCamelCase_							  =					word_embeddings

							# It should be done!
							return output_state_dict



def      __lowerCAmelCase						(							)  ->    List[Any]:
							# Create the argument parser.
							lowerCamelCase_							  =					argparse.ArgumentParser()
							parser.add_argument("""--print-checkpoint-structure"""			,				action="""store_true"""       )
							parser.add_argument(
							    """path_to_checkpoint"""			,				type=UpperCAmelCase__			,				help="""Path to the checkpoint file (.zip archive or direct .pt file)"""			,				)
							parser.add_argument(
							    """--config_file"""			,				default=""""""			,				type=UpperCAmelCase__			,				help="""An optional config json file describing the pre-trained model."""			,				)
							lowerCamelCase_							  =					parser.parse_args()

							# Extract the basename.
							lowerCamelCase_							  =					os.path.dirname(args.path_to_checkpoint       )

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''       )
							if args.path_to_checkpoint.endswith(""".zip"""       ):
														with zipfile.ZipFile(args.path_to_checkpoint			,				"""r"""       ) as checkpoint:
																					with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""       ) as pytorch_dict:
																												lowerCamelCase_							  =					torch.load(UpperCAmelCase__			,				map_location="""cpu"""       )
							else:
														lowerCamelCase_							  =					torch.load(args.path_to_checkpoint			,				map_location="""cpu"""       )

							lowerCamelCase_							  =					input_state_dict.get("""args"""			,				UpperCAmelCase__       )

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												lowerCamelCase_							  =					"""gelu_fast"""
																					elif ds_args.openai_gelu:
																												lowerCamelCase_							  =					"""gelu_new"""
																					else:
																												lowerCamelCase_							  =					"""gelu"""
														else:
																					# in the very early days this used to be "gelu_new"
																					lowerCamelCase_							  =					"""gelu_new"""

														# Spell out all parameters in case the defaults change.
														lowerCamelCase_							  =					GPTaConfig(
														    vocab_size=5_0_2_5_7			,				n_positions=1_0_2_4			,				n_embd=1_0_2_4			,				n_layer=2_4			,				n_head=1_6			,				n_inner=4_0_9_6			,				activation_function=UpperCAmelCase__			,				resid_pdrop=0.1			,				embd_pdrop=0.1			,				attn_pdrop=0.1			,				layer_norm_epsilon=1e-5			,				initializer_range=0.0_2			,				summary_type="""cls_index"""			,				summary_use_proj=UpperCAmelCase__			,				summary_activation=UpperCAmelCase__			,				summary_proj_to_labels=UpperCAmelCase__			,				summary_first_dropout=0.1			,				scale_attn_weights=UpperCAmelCase__			,				use_cache=UpperCAmelCase__			,				bos_token_id=5_0_2_5_6			,				eos_token_id=5_0_2_5_6			,				)
							else:
														lowerCamelCase_							  =					GPTaConfig.from_json_file(args.config_file       )

							lowerCamelCase_							  =					["""GPT2LMHeadModel"""]

							# Convert.
							print("""Converting"""       )
							lowerCamelCase_							  =					convert_megatron_checkpoint(UpperCAmelCase__			,				UpperCAmelCase__			,				UpperCAmelCase__       )

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(UpperCAmelCase__			,				UpperCAmelCase__       )

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														lowerCamelCase_							  =					ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					lowerCamelCase_							  =					"""gpt2"""
														elif tokenizer_type == "PretrainedFromHF":
																					lowerCamelCase_							  =					ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''       )
							else:
														lowerCamelCase_							  =					"""gpt2"""

							lowerCamelCase_							  =					AutoTokenizer.from_pretrained(UpperCAmelCase__       )
							lowerCamelCase_							  =					type(UpperCAmelCase__       ).__name__
							lowerCamelCase_							  =					tokenizer_class

							# Store the config to file.
							print("""Saving config"""       )
							config.save_pretrained(UpperCAmelCase__       )

							# Save tokenizer based on args
							print(F'''Adding {tokenizer_class} tokenizer files'''       )
							tokenizer.save_pretrained(UpperCAmelCase__       )

							# Store the state_dict to file.
							lowerCamelCase_							  =					os.path.join(UpperCAmelCase__			,				"""pytorch_model.bin"""       )
							print(F'''Saving checkpoint to "{output_checkpoint_file}"'''       )
							torch.save(UpperCAmelCase__			,				UpperCAmelCase__       )


####################################################################################################

if __name__ == "__main__":
		main()

####################################################################################################







import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase        =  get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase        =  2_5_0_0_0_4
lowercase        =  2_5_0_0_2_0





@require_sentencepiece
@require_tokenizers
class       __A(				UpperCAmelCase     ,		unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizer
					SCREAMING_SNAKE_CASE			    =    MBartaaTokenizerFast
					SCREAMING_SNAKE_CASE			    =    True
					SCREAMING_SNAKE_CASE			    =    True

					def    lowercase__      (	self					:     Tuple   ):
												super().setUp()

												# We have a SentencePiece fixture for testing
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )
												tokenizer.save_pretrained(self.tmpdirname   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					"""<s>"""
												lowerCamelCase_							  =					0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase   )   ,					__UpperCamelCase   )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					list(self.get_tokenizer().get_vocab().keys()   )

												self.assertEqual(vocab_keys[0]   ,					"""<s>"""   )
												self.assertEqual(vocab_keys[1]   ,					"""<pad>"""   )
												self.assertEqual(vocab_keys[-1]   ,					"""<mask>"""   )
												self.assertEqual(len(__UpperCamelCase   )   ,					1_0_5_4   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												self.assertEqual(self.get_tokenizer().vocab_size   ,					1_0_5_4   )

					def    lowercase__      (	self					:     str   ):
												lowerCamelCase_							  =					MBartaaTokenizer(__UpperCamelCase   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   ,					keep_accents=__UpperCamelCase   )

												lowerCamelCase_							  =					tokenizer.tokenize("""This is a test"""   )
												self.assertListEqual(__UpperCamelCase   ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]   )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__UpperCamelCase   )   ,					[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]   ,					)

												lowerCamelCase_							  =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]   ,					)
												lowerCamelCase_							  =					tokenizer.convert_tokens_to_ids(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[
												        value + tokenizer.fairseq_offset
												        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
												    ]   ,					)

												lowerCamelCase_							  =					tokenizer.convert_ids_to_tokens(__UpperCamelCase   )
												self.assertListEqual(
												    __UpperCamelCase   ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]   ,					)

					@slow
					def    lowercase__      (	self					:     Optional[int]   ):
												# fmt: off
												lowerCamelCase_							  =					{"""input_ids""": [[2_5_0_0_0_4, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [2_5_0_0_0_4, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_0_0_0_4, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=__UpperCamelCase   ,					model_name="""facebook/mbart-large-50"""   ,					revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""   ,					)



					def    lowercase__      (	self					:     Dict   ):
												if not self.test_slow_tokenizer:
																			# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																			return

												lowerCamelCase_							  =					(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																										lowerCamelCase_							  =					self.rust_tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )
																										lowerCamelCase_							  =					self.tokenizer_class.from_pretrained(__UpperCamelCase   ,					**__UpperCamelCase   )

																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files + the tokenizer.json file for the fast one
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )
																										lowerCamelCase_							  =					tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f   )
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )
																																	# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																																	# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=True
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it save with the same files
																										self.assertSequenceEqual(__UpperCamelCase   ,					__UpperCamelCase   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )

																										# Save tokenizer rust, legacy_format=False
																										lowerCamelCase_							  =					tempfile.mkdtemp()

																										lowerCamelCase_							  =					tokenizer_r.save_pretrained(__UpperCamelCase   ,					legacy_format=__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.save_pretrained(__UpperCamelCase   )

																										# Checks it saved the tokenizer.json file
																										self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )

																										# Checks everything loads correctly in the same way
																										lowerCamelCase_							  =					tokenizer_r.from_pretrained(__UpperCamelCase   )
																										lowerCamelCase_							  =					tokenizer_p.from_pretrained(__UpperCamelCase   )

																										# Check special tokens are set accordingly on Rust and Python
																										for key in tokenizer_pp.special_tokens_map:
																																	self.assertTrue(hasattr(__UpperCamelCase   ,					__UpperCamelCase   )   )

																										shutil.rmtree(__UpperCamelCase   )





@require_torch
@require_sentencepiece
@require_tokenizers
class       __A(				unittest.TestCase       ):
					SCREAMING_SNAKE_CASE			    =    '''facebook/mbart-large-50-one-to-many-mmt'''
					SCREAMING_SNAKE_CASE			    =    [
					    ''' UN Chief Says There Is No Military Solution in Syria''',
					    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
					]
					SCREAMING_SNAKE_CASE			    =    [
					    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
					    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
					    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
					    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
					]
					SCREAMING_SNAKE_CASE			    =    [EN_CODE, 8_2_7_4, 1_2_7_8_7_3, 2_5_9_1_6, 7, 8_6_2_2, 2_0_7_1, 4_3_8, 6_7_4_8_5, 5_3, 1_8_7_8_9_5, 2_3, 5_1_7_1_2, 2]

					@classmethod
					def    lowercase__      (	cls					:     Optional[Any]   ):
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(
												    cls.checkpoint_name   ,					src_lang="""en_XX"""   ,					tgt_lang="""ro_RO"""   )
												lowerCamelCase_							  =					1
												return cls

					def    lowercase__      (	self					:     Union[str, Any]   ):
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]   ,					2_5_0_0_0_1   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]   ,					2_5_0_0_0_4   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]   ,					2_5_0_0_2_0   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]   ,					2_5_0_0_3_8   )

					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Tuple   ):
												self.assertIn(__UpperCamelCase   ,					self.tokenizer.all_special_ids   )
												lowerCamelCase_							  =					[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
												lowerCamelCase_							  =					self.tokenizer.decode(__UpperCamelCase   ,					skip_special_tokens=__UpperCamelCase   )
												lowerCamelCase_							  =					self.tokenizer.decode(generated_ids[1:]   ,					skip_special_tokens=__UpperCamelCase   )
												self.assertEqual(__UpperCamelCase   ,					__UpperCamelCase   )
												self.assertNotIn(self.tokenizer.eos_token   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												lowerCamelCase_							  =					["""this is gunna be a long sentence """ * 2_0]
												assert isinstance(src_text[0]   ,					__UpperCamelCase   )
												lowerCamelCase_							  =					1_0
												lowerCamelCase_							  =					self.tokenizer(__UpperCamelCase   ,					max_length=__UpperCamelCase   ,					truncation=__UpperCamelCase   ).input_ids[0]
												self.assertEqual(ids[0]   ,					__UpperCamelCase   )
												self.assertEqual(ids[-1]   ,					2   )
												self.assertEqual(len(__UpperCamelCase   )   ,					__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ):
												self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]   )   ,					[2_5_0_0_5_3, 2_5_0_0_0_1]   )

					def    lowercase__      (	self					:     Dict   ):
												lowerCamelCase_							  =					tempfile.mkdtemp()
												lowerCamelCase_							  =					self.tokenizer.fairseq_tokens_to_ids
												self.tokenizer.save_pretrained(__UpperCamelCase   )
												lowerCamelCase_							  =					MBartaaTokenizer.from_pretrained(__UpperCamelCase   )
												self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,					__UpperCamelCase   )

					@require_torch
					def    lowercase__      (	self					:     Optional[Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
												assert batch.input_ids[1][0] == EN_CODE
												assert batch.input_ids[1][-1] == 2
												assert batch.labels[1][0] == RO_CODE
												assert batch.labels[1][-1] == 2
												assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]

					@require_torch
					def    lowercase__      (	self					:     Optional[int]   ):
												lowerCamelCase_							  =					self.tokenizer(
												    self.src_text   ,					text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=len(self.expected_src_tokens   )   ,					return_tensors="""pt"""   ,					)
												lowerCamelCase_							  =					shift_tokens_right(batch["""labels"""]   ,					self.tokenizer.pad_token_id   )

												self.assertIsInstance(__UpperCamelCase   ,					__UpperCamelCase   )

												self.assertEqual((2, 1_4)   ,					batch.input_ids.shape   )
												self.assertEqual((2, 1_4)   ,					batch.attention_mask.shape   )
												lowerCamelCase_							  =					batch.input_ids.tolist()[0]
												self.assertListEqual(self.expected_src_tokens   ,					__UpperCamelCase   )
												self.assertEqual(2   ,					batch.decoder_input_ids[0, 0]   )  # decoder_start_token_id
												# Test that special tokens are reset
												self.assertEqual(self.tokenizer.prefix_tokens   ,					[EN_CODE]   )
												self.assertEqual(self.tokenizer.suffix_tokens   ,					[self.tokenizer.eos_token_id]   )

					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer(self.src_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=3   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					self.tokenizer(
												    text_target=self.tgt_text   ,					padding=__UpperCamelCase   ,					truncation=__UpperCamelCase   ,					max_length=1_0   ,					return_tensors="""pt"""   )
												lowerCamelCase_							  =					targets["""input_ids"""]
												lowerCamelCase_							  =					shift_tokens_right(__UpperCamelCase   ,					self.tokenizer.pad_token_id   )

												self.assertEqual(batch.input_ids.shape[1]   ,					3   )
												self.assertEqual(batch.decoder_input_ids.shape[1]   ,					1_0   )



					@require_torch
					def    lowercase__      (	self					:     Union[str, Any]   ):
												lowerCamelCase_							  =					self.tokenizer._build_translation_inputs(
												    """A test"""   ,					return_tensors="""pt"""   ,					src_lang="""en_XX"""   ,					tgt_lang="""ar_AR"""   )

												self.assertEqual(
												    nested_simplify(__UpperCamelCase   )   ,					{
												        # en_XX, A, test, EOS
												        """input_ids""": [[2_5_0_0_0_4, 6_2, 3_0_3_4, 2]],
												        """attention_mask""": [[1, 1, 1, 1]],
												        # ar_AR
												        """forced_bos_token_id""": 2_5_0_0_0_1,
												    }   ,					)





