



import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowercase_													=      os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(F"{bindir}/../../examples/pytorch/translation"):
				from run_translation import main  # noqa


set_seed(42)
lowercase_													=      '''sshleifer/student_marian_en_ro_6_1'''
lowercase_													=      '''sshleifer/tiny-mbart'''






@require_torch
class     __a			(  SCREAMING_SNAKE_CASE					):



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     Any=False					,						snake_case_				:     Dict=None					,						snake_case_				:     str=True					,						snake_case_				:     Union[str, Any]=True					,						snake_case_				:     List[Any]=True					,						snake_case_				:     Any=True					,						)->				List[Any]:
						__lowerCAmelCase						 =self.run_trainer(
						    eval_steps=1					,						max_len=12					,						model_name=snake_case_					,						num_train_epochs=1					,						distributed=snake_case_					,						extra_args_str=snake_case_					,						predict_with_generate=snake_case_					,						do_train=snake_case_					,						do_eval=snake_case_					,						do_predict=snake_case_					,						)
						__lowerCAmelCase						 =TrainerState.load_from_json(os.path.join(snake_case_					,						"""trainer_state.json""")).log_history

						if not do_eval:
											return

						__lowerCAmelCase						 =[log for log in logs if """eval_loss""" in log.keys()]

						__lowerCAmelCase						 =eval_metrics[0]
						if predict_with_generate:
											assert "eval_bleu" in first_step_stats

											__lowerCAmelCase						 =eval_metrics[-1]
											assert isinstance(last_step_stats["""eval_bleu"""]					,						snake_case_)
											assert not math.isnan(float(last_step_stats["""eval_loss"""])), "eval_loss must not be `nan`"



	@require_torch_non_multi_gpu
	def 		UpperCamelCase				(    self				:     int)->				Tuple:
						self.run_seqaseq_quick()



	@require_torch_multi_gpu
	def 		UpperCamelCase				(    self				:     Tuple)->				Optional[int]:
						self.run_seqaseq_quick(distributed=snake_case_)



	@require_torch_multi_gpu
	def 		UpperCamelCase				(    self				:     str)->				Union[str, Any]:
						self.run_seqaseq_quick(distributed=snake_case_)



	@unittest.skip("""Requires an update of the env running those tests""")
	@require_torch_multi_gpu
	@require_fairscale
	def 		UpperCamelCase				(    self				:     Optional[int])->				Optional[int]:
						self.run_seqaseq_quick(distributed=snake_case_					,						extra_args_str="""--sharded_ddp simple""")



	@unittest.skip("""Requires an update of the env running those tests""")
	@require_torch_multi_gpu
	@require_fairscale
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				str:
						self.run_seqaseq_quick(distributed=snake_case_					,						extra_args_str="""--sharded_ddp simple --fp16""")



	@unittest.skip("""Requires an update of the env running those tests""")
	@require_torch_multi_gpu
	@require_fairscale
	def 		UpperCamelCase				(    self				:     Optional[int])->				Any:
						self.run_seqaseq_quick(distributed=snake_case_					,						extra_args_str="""--sharded_ddp zero_dp_2"""					,						predict_with_generate=snake_case_)



	@unittest.skip("""Requires an update of the env running those tests""")
	@require_torch_multi_gpu
	@require_fairscale
	def 		UpperCamelCase				(    self				:     int)->				Union[str, Any]:
						self.run_seqaseq_quick(
						    distributed=snake_case_					,						extra_args_str="""--sharded_ddp zero_dp_2 --fp16"""					,						predict_with_generate=snake_case_)



	@require_apex
	@require_torch_gpu
	def 		UpperCamelCase				(    self				:     Optional[int])->				str:
						# XXX: apex breaks the trainer if it's run twice e.g. run_seq2seq.main() from the same
						# program and it breaks other tests that run from the same pytest worker, therefore until this is
						# sorted out it must be run only in an external program, that is distributed=True in this
						# test and only under one or more gpus - if we want cpu will need to make a special test
						#
						# specifically to the problem traced it to self.optimizer.step() - if it's run 2nd time via
						# 2nd main() call it botches the future eval.
						#
						self.run_seqaseq_quick(distributed=snake_case_					,						extra_args_str="""--fp16 --fp16_backend=apex""")
						# test 2nd time - was getting eval_loss': nan'
						# to reproduce the problem set distributed=False
						self.run_seqaseq_quick(distributed=snake_case_					,						extra_args_str="""--fp16 --fp16_backend=apex""")



	@parameterized.expand(["""base""", """low""", """high""", """mixed"""])
	@require_torch_multi_gpu
	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     Dict)->				Dict:
						# as each sub-test is slow-ish split into multiple sub-tests to avoid CI timeout
						__lowerCAmelCase						 ={
						    # test with the default log_level - should be info and thus log info once
						    """base""": {"""extra_args_str""": """""", """n_matches""": 1},
						    # test with low log_level and log_level_replica - should be noisy on all processes
						    # now the info string should appear twice on 2 processes
						    """low""": {"""extra_args_str""": """--log_level debug --log_level_replica debug""", """n_matches""": 2},
						    # test with high log_level and low log_level_replica
						    # now the info string should appear once only on the replica
						    """high""": {"""extra_args_str""": """--log_level error --log_level_replica debug""", """n_matches""": 1},
						    # test with high log_level and log_level_replica - should be quiet on all processes
						    """mixed""": {"""extra_args_str""": """--log_level error --log_level_replica error""", """n_matches""": 0},
						}

						__lowerCAmelCase						 =experiments[experiment_id]
						__lowerCAmelCase						 ={"""distributed""": True, """predict_with_generate""": False, """do_eval""": False, """do_predict""": False}
						__lowerCAmelCase						 ="""Running training"""
						with CaptureStderr() as cl:
											self.run_seqaseq_quick(**snake_case_					,						extra_args_str=data["""extra_args_str"""])
						__lowerCAmelCase						 =len(re.findall(snake_case_					,						cl.err))
						self.assertEqual(snake_case_					,						data["""n_matches"""])



	@slow
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Tuple:
						__lowerCAmelCase						 =self.run_trainer(
						    eval_steps=2					,						max_len=1_28					,						model_name=snake_case_					,						learning_rate=3e-4					,						num_train_epochs=10					,						distributed=snake_case_					,						)

						# Check metrics
						__lowerCAmelCase						 =TrainerState.load_from_json(os.path.join(snake_case_					,						"""trainer_state.json""")).log_history
						__lowerCAmelCase						 =[log for log in logs if """eval_loss""" in log.keys()]
						__lowerCAmelCase						 =eval_metrics[0]
						__lowerCAmelCase						 =eval_metrics[-1]

						assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
						assert isinstance(last_step_stats["""eval_bleu"""]					,						snake_case_)

						# test if do_predict saves generations and metrics
						__lowerCAmelCase						 =os.listdir(snake_case_)
						__lowerCAmelCase						 ={os.path.basename(snake_case_) for p in contents}
						assert "generated_predictions.txt" in contents
						assert "predict_results.json" in contents



	@slow
	@require_bitsandbytes
	def 		UpperCamelCase				(    self				:     List[Any])->				Any:
						from transformers.training_args import OptimizerNames

						def train_and_return_metrics(snake_case_				:     str) -> Tuple[int, float]:
											__lowerCAmelCase						 ="""--skip_memory_metrics 0"""

											__lowerCAmelCase						 =self.run_trainer(
											    max_len=1_28					,						model_name=snake_case_					,						learning_rate=3e-4					,						num_train_epochs=1					,						optim=snake_case_					,						distributed=snake_case_					,						extra_args_str=snake_case_					,						do_eval=snake_case_					,						do_predict=snake_case_					,						n_gpus_to_use=1					,						)

											# Check metrics
											__lowerCAmelCase						 =TrainerState.load_from_json(Path(snake_case_					,						"""trainer_state.json""")).log_history
											__lowerCAmelCase						 =int(logs[0]["""train_mem_gpu_peaked_delta"""] / 2**20)
											__lowerCAmelCase						 =int(logs[0]["""train_mem_gpu_alloc_delta"""] / 2**20)

											__lowerCAmelCase						 =logs[0]["""train_loss"""]
											return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase						 =train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value)
						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase						 =train_and_return_metrics(OptimizerNames.ADAMW_BNB.value)

						__lowerCAmelCase						 =gpu_alloc_mem_orig - gpu_alloc_mem_bnb

						__lowerCAmelCase						 =gpu_peak_mem_orig + gpu_alloc_mem_orig
						__lowerCAmelCase						 =gpu_peak_mem_bnb + gpu_alloc_mem_bnb
						__lowerCAmelCase						 =gpu_total_mem_orig - gpu_total_mem_bnb

						# sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
						# doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
						# in 2 bytes and the diff in optim memory usage is derived as so:
						#
						# - normal 25*8=~200MB (8 bytes per param)
						# - bnb    25*2= ~50MB (2 bytes per param)
						#
						# Thus we should expect ~150MB total memory saved.
						#
						# Peak memory should be the same - the total should be different by about that same margin
						#
						# After leaving a small margin to accommodate for differences between gpus let's check
						# that we have at least 120MB in savings
						__lowerCAmelCase						 =1_20

						# uncomment the following if this test starts failing - requires py38 for a new print feature
						# gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
						# print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
						# print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
						# print(f"{gpu_alloc_mem_diff=}MB")
						# print(f"{gpu_peak_mem_diff=}MB")
						# print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
						# print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

						self.assertGreater(
						    snake_case_					,						snake_case_					,						"""should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"""
						    F""" a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and"""
						    F""" gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB"""					,						)

						self.assertGreater(
						    snake_case_					,						snake_case_					,						"""should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"""
						    F""" a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and"""
						    F""" gpu_total_mem_bnb={gpu_total_mem_bnb}MB"""					,						)

						self.assertEqual(
						    snake_case_					,						snake_case_					,						F"""loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}""")



	def 		UpperCamelCase				(    self				:     List[str]					,						snake_case_				:     int					,						snake_case_				:     str					,						snake_case_				:     int					,						snake_case_				:     float = 3e-3					,						snake_case_				:     str = "adafactor"					,						snake_case_				:     bool = False					,						snake_case_				:     str = None					,						snake_case_				:     int = 0					,						snake_case_				:     bool = True					,						snake_case_				:     bool = True					,						snake_case_				:     bool = True					,						snake_case_				:     bool = True					,						snake_case_				:     int = None					,						)->				Optional[int]:
						__lowerCAmelCase						 =self.test_file_dir / """../fixtures/tests_samples/wmt_en_ro"""
						__lowerCAmelCase						 =self.get_auto_remove_tmp_dir()
						__lowerCAmelCase						 =F"""
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(snake_case_)}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(snake_case_)}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        """.split()

						__lowerCAmelCase						 =F"""
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(snake_case_)}
        """.split()

						__lowerCAmelCase						 ="""
            --do_predict
        """.split()

						__lowerCAmelCase						 =[]
						if do_train:
											args += args_train

						if do_eval:
											args += args_eval

						if do_predict:
											args += args_predict

						if predict_with_generate:
											args += "--predict_with_generate".split()

						if do_train:
											if optim == "adafactor":
																args += "--adafactor".split()
											else:
																args += F"""--optim {optim}""".split()

						if extra_args_str is not None:
											args += extra_args_str.split()

						if distributed:
											if n_gpus_to_use is None:
																__lowerCAmelCase						 =get_gpu_count()
											__lowerCAmelCase						 =get_torch_dist_unique_port()
											__lowerCAmelCase						 =F"""
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            """.split()
											__lowerCAmelCase						 =[sys.executable] + distributed_args + args
											# keep for quick debug
											# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
											execute_subprocess_async(snake_case_					,						env=self.get_env())
						else:
											__lowerCAmelCase						 =["""run_translation.py"""] + args
											with patch.object(snake_case_					,						"""argv"""					,						snake_case_):
																main()

						return output_dir









from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


lowercase_													=      logging.get_logger(__name__)

lowercase_													=      {
    '''EleutherAI/gpt-neo-1.3B''': '''https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json''',
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}






class     __a			(  SCREAMING_SNAKE_CASE					):
	SCREAMING_SNAKE_CASE  	=							"gpt_neo"
	SCREAMING_SNAKE_CASE  	=							["past_key_values"]
	SCREAMING_SNAKE_CASE  	=							{"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}



	def __init__(    self				:     Optional[int]					,						snake_case_				:     int=5_02_57					,						snake_case_				:     Tuple=20_48					,						snake_case_				:     Optional[int]=20_48					,						snake_case_				:     int=24					,						snake_case_				:     List[Any]=[[["global", "local"], 12]]					,						snake_case_				:     Tuple=16					,						snake_case_				:     Optional[int]=None					,						snake_case_				:     Dict=2_56					,						snake_case_				:     List[str]="gelu_new"					,						snake_case_				:     List[Any]=0.0					,						snake_case_				:     Tuple=0.0					,						snake_case_				:     Tuple=0.0					,						snake_case_				:     str=0.1					,						snake_case_				:     Union[str, Any]=1e-5					,						snake_case_				:     str=0.0_2					,						snake_case_				:     Union[str, Any]=True					,						snake_case_				:     Optional[int]=5_02_56					,						snake_case_				:     Tuple=5_02_56					,						**snake_case_				:     Optional[int]					,						)->				Tuple:
						__lowerCAmelCase						 =vocab_size
						__lowerCAmelCase						 =max_position_embeddings
						__lowerCAmelCase						 =hidden_size
						__lowerCAmelCase						 =num_layers
						__lowerCAmelCase						 =num_heads
						__lowerCAmelCase						 =intermediate_size
						__lowerCAmelCase						 =window_size
						__lowerCAmelCase						 =activation_function
						__lowerCAmelCase						 =resid_dropout
						__lowerCAmelCase						 =embed_dropout
						__lowerCAmelCase						 =attention_dropout
						__lowerCAmelCase						 =classifier_dropout
						__lowerCAmelCase						 =layer_norm_epsilon
						__lowerCAmelCase						 =initializer_range
						__lowerCAmelCase						 =use_cache

						__lowerCAmelCase						 =bos_token_id
						__lowerCAmelCase						 =eos_token_id

						__lowerCAmelCase						 =attention_types
						__lowerCAmelCase						 =self.expand_attention_types_params(snake_case_)

						if len(self.attention_layers) != self.num_layers:
											raise ValueError(
											    """Configuration for convolutional module is incorrect. """
											    """It is required that `len(config.attention_layers)` == `config.num_layers` """
											    F"""but is `len(config.attention_layers) = {len(self.attention_layers)}`, """
											    F"""`config.num_layers = {self.num_layers}`. """
											    """`config.attention_layers` is prepared using `config.attention_types`. """
											    """Please verify the value of `config.attention_types` argument.""")

						super().__init__(bos_token_id=snake_case_					,						eos_token_id=snake_case_					,						**snake_case_)



	@staticmethod
	def 		UpperCamelCase				(    snake_case_				:     Optional[int])->				Any:
						__lowerCAmelCase						 =[]
						for item in attention_types:
											for _ in range(item[1]):
																attentions.extend(item[0])
						return attentions






def        __lowerCAmelCase		(	__lowerCamelCase						:     Any					, __lowerCamelCase						:     List[str]					, __lowerCamelCase						:     Optional[Any]					, __lowerCamelCase						:     int						)	->    int:
					import torch

					__lowerCAmelCase						 =input.size()
					__lowerCAmelCase						 =len(__lowerCamelCase						)
					__lowerCAmelCase						 =shape[dimension]

					__lowerCAmelCase						 =torch.arange(0					, __lowerCamelCase					, __lowerCamelCase						)
					__lowerCAmelCase						 =torch.div(sizedim - size					, __lowerCamelCase					, rounding_mode="""floor"""						) + 1
					__lowerCAmelCase						 =torch.arange(__lowerCamelCase						) + low_indices[:min_length][:, None]

					__lowerCAmelCase						 =[slice(__lowerCamelCase						)] * rank
					__lowerCAmelCase						 =indices
					__lowerCAmelCase						 =input[s]

					__lowerCAmelCase						 =list(range(0					, rank + 1						)						)
					perm.append(perm.pop(dimension + 1						)						)

					return sliced.permute(__lowerCamelCase						)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Optional[Any]					, __lowerCamelCase						:     Any						)	->    List[str]:
					import torch

					__lowerCAmelCase						 =torch.arange(1					, __lowerCamelCase						)
					__lowerCAmelCase						 =torch.remainder(__lowerCamelCase					, __lowerCamelCase						)
					__lowerCAmelCase						 =remainders == 0
					__lowerCAmelCase						 =candidates[divisor_indices]
					__lowerCAmelCase						 =torch.max(__lowerCamelCase						)
					return largest_divisor, torch.div(__lowerCamelCase					, __lowerCamelCase					, rounding_mode="""floor"""						)






class     __a			(  SCREAMING_SNAKE_CASE					):



	@property
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Mapping[str, Mapping[int, str]]:
						__lowerCAmelCase						 =OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}})
						if self.use_past:
											self.fill_with_past_key_values_(snake_case_					,						direction="""inputs""")
											__lowerCAmelCase						 ={0: """batch""", 1: """past_sequence + sequence"""}
						else:
											__lowerCAmelCase						 ={0: """batch""", 1: """sequence"""}

						return common_inputs



	@property
	def 		UpperCamelCase				(    self				:     Optional[int])->				int:
						return self._config.num_heads



	def 		UpperCamelCase				(    self				:     int					,						snake_case_				:     PreTrainedTokenizer					,						snake_case_				:     int = -1					,						snake_case_				:     int = -1					,						snake_case_				:     bool = False					,						snake_case_				:     Optional[TensorType] = None					,						)->				Mapping[str, Any]:
						__lowerCAmelCase						 =super(snake_case_					,						self).generate_dummy_inputs(
						    snake_case_					,						batch_size=snake_case_					,						seq_length=snake_case_					,						is_pair=snake_case_					,						framework=snake_case_)

						# We need to order the input in the way they appears in the forward()
						__lowerCAmelCase						 =OrderedDict({"""input_ids""": common_inputs["""input_ids"""]})

						# Need to add the past_keys
						if self.use_past:
											if not is_torch_available():
																raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed.""")
											else:
																import torch

																__lowerCAmelCase ,						__lowerCAmelCase						 =common_inputs["""input_ids"""].shape
																# Not using the same length for past_key_values
																__lowerCAmelCase						 =seqlen + 2
																__lowerCAmelCase						 =(
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																__lowerCAmelCase						 =[
																    (torch.zeros(snake_case_), torch.zeros(snake_case_)) for _ in range(self.num_layers)
																]

						__lowerCAmelCase						 =common_inputs["""attention_mask"""]
						if self.use_past:
											__lowerCAmelCase						 =ordered_inputs["""attention_mask"""].dtype
											__lowerCAmelCase						 =torch.cat(
											    [ordered_inputs["""attention_mask"""], torch.ones(snake_case_					,						snake_case_					,						dtype=snake_case_)]					,						dim=1)

						return ordered_inputs



	@property
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				int:
						return 13




