


from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


lowercase  :							str			  =    logging.get_logger(__name__)


if is_torch_available():
							import torch
							from torch import nn
							from torch.utils.data import Dataset

							from transformers import Trainer




							class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


													'''simple docstring'''




													def __init__(	self		,				_SCREAMING_SNAKE_CASE = 101				) ->				str:
															snake_case_				:							Dict						 =       length






													def __len__(	self				) ->				str:
															return self.length






													def __getitem__(	self		,				_SCREAMING_SNAKE_CASE				) ->				int:
															return i




							class 		UpperCAmelCase_						:


													'''simple docstring'''




													def __call__(	self		,				_SCREAMING_SNAKE_CASE				) ->				List[str]:
															return {"input_ids": torch.tensor(_SCREAMING_SNAKE_CASE				), "labels": torch.tensor(_SCREAMING_SNAKE_CASE				)}




							class 		UpperCAmelCase_						(	nn.Module					):


													'''simple docstring'''




													def __init__(	self				) ->				Any:
															super().__init__()
															# Add some (unused) params otherwise DDP will complain.
															snake_case_				:							List[Any]						 =       nn.Linear(120		,				80				)






													def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=None				) ->				int:
															if labels is not None:
																	return torch.tensor(0.0		,				device=input_ids.device				), input_ids
															else:
																	return input_ids




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						@require_torch_neuroncore
						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							Optional[int]						 =       f'''--nproc_per_node=2
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        '''.split()
								snake_case_				:							Any						 =       self.get_auto_remove_tmp_dir()
								snake_case_				:							List[Any]						 =       f'''--output_dir {output_dir}'''.split()
								snake_case_				:							List[Any]						 =       ["torchrun"] + distributed_args + args
								execute_subprocess_async(_SCREAMING_SNAKE_CASE		,				env=self.get_env()				)
								# successful return here == success - any errors would have caused an error in the sub-call




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						@require_torch_multi_gpu
						def       _lowerCAmelCase       (	self				) ->				Dict:
								snake_case_				:							str						 =       f'''--nproc_per_node={torch.cuda.device_count()}
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        '''.split()
								snake_case_				:							Tuple						 =       self.get_auto_remove_tmp_dir()
								snake_case_				:							Optional[Any]						 =       f'''--output_dir {output_dir}'''.split()
								snake_case_				:							str						 =       ["torchrun"] + distributed_args + args
								execute_subprocess_async(_SCREAMING_SNAKE_CASE		,				env=self.get_env()				)
								# successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
							# The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
							#
							# PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

							lowercase  :							Optional[int]			  =    HfArgumentParser((TrainingArguments,))
							lowercase  :							Any			  =    parser.parse_args_into_dataclasses()[0]

							logger.warning(
							    F"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, """
							    F"""distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}"""
							)

							# Essentially, what we want to verify in the distributed case is that we get all samples back,
							# in the right order. (this is crucial for prediction for instance)
							for dataset_length in [1_01, 40, 7]:
														lowercase  :							Optional[Any]			  =    DummyDataset(dataset_length)

														def      lowerCAmelCase__	(    _a       :			EvalPrediction    ):
																snake_case_				:							Any						 =       list(range(len(_a    )    )    )
																snake_case_				:							str						 =       p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
																if not success and training_args.local_rank == 0:
																		logger.warning(
																		    "Predictions and/or labels do not match expected results:\n  - predictions: "
																		    F'''{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}'''    )
																return {"success": success}

														lowercase  :							List[Any]			  =    Trainer(
														    model=DummyModel(),
														    args=training_args,
														    data_collator=DummyDataCollator(),
														    eval_dataset=dataset,
														    compute_metrics=compute_metrics,
														)
														lowercase  :							Dict			  =    trainer.evaluate()
														logger.info(metrics)
														if metrics["eval_success"] is not True:
																					logger.error(metrics)
																					exit(1)

														lowercase  :							str			  =    trainer.predict(dataset)
														logger.info(p.metrics)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics)
																					exit(1)

														lowercase  :							Tuple			  =    2

														lowercase  :							List[Any]			  =    trainer.evaluate()
														logger.info(metrics)
														if metrics["eval_success"] is not True:
																					logger.error(metrics)
																					exit(1)

														lowercase  :							List[str]			  =    trainer.predict(dataset)
														logger.info(p.metrics)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics)
																					exit(1)

														lowercase  :							str			  =    None








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase  :							Optional[int]			  =    {
    '''configuration_nllb_moe''': [
        '''NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''NllbMoeConfig''',
    ]
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase  :							Optional[Any]			  =    [
							    '''NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''NllbMoeForConditionalGeneration''',
							    '''NllbMoeModel''',
							    '''NllbMoePreTrainedModel''',
							    '''NllbMoeTop2Router''',
							    '''NllbMoeSparseMLP''',
							]


if TYPE_CHECKING:
							from .configuration_nllb_moe import (
							    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    NllbMoeConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_nllb_moe import (
														    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    NllbMoeForConditionalGeneration,
														    NllbMoeModel,
														    NllbMoePreTrainedModel,
														    NllbMoeSparseMLP,
														    NllbMoeTopaRouter,
														)


else:
							import sys

							lowercase  :							Tuple			  =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
