
import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
					from sagemaker import Session, TrainingJobAnalytics
					from sagemaker.huggingface import HuggingFace

@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'      ,	'False'					)					) is not True      ,	reason='Skipping test because should only be run when releasing minor transformers version'      ,	)
@pytest.mark.usefixtures('sm_env'					)
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 650, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'pytorch',
            'script': 'run_ddp.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf_dist.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.6, 'eval_loss': 0.7},
        },
    ]					)
class       a		(   unittest.TestCase					):







							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								if self.framework == "pytorch":
									subprocess.run(
									    F"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()      ,				encoding="utf-8"      ,				check=snake_case__      ,				)
								assert hasattr(self      ,				"env" )







							def       UpperCAmelCase__					(						self  :  Optional[int]      ,				snake_case__  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							F"{self.env.base_job_name}-{instance_count}-{'ddp' if 'ddp' in self.script else 'smd'}"
								# distributed data settings
								__lowerCAmelCase         =							{"smdistributed": {"dataparallel": {"enabled": True}}} if self.script != "run_ddp.py" else None

								# creates estimator
								return HuggingFace(
								    entry_point=self.script      ,				source_dir=self.env.test_path      ,				role=self.env.role      ,				image_uri=self.env.image_uri      ,				base_job_name=snake_case__      ,				instance_count=snake_case__      ,				instance_type=self.instance_type      ,				debugger_hook_config=snake_case__      ,				hyperparameters={**self.env.distributed_hyperparameters, "model_name_or_path": self.model_name_or_path}      ,				metric_definitions=self.env.metric_definitions      ,				distribution=snake_case__      ,				py_version="py36"      ,				)







							def       UpperCAmelCase__					(						self  :  List[Any]      ,				snake_case__  :  Optional[int] ):
								"""simple docstring"""







								TrainingJobAnalytics(snake_case__ ).export_csv(F"{self.env.test_path}/{job_name}_metrics.csv" )







							@parameterized.expand([(2,)] )
							def       UpperCAmelCase__					(						self  :  Dict      ,				snake_case__  :  List[str] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.create_estimator(snake_case__ )

								# run training
								estimator.fit()

								# result dataframe
								__lowerCAmelCase         =							TrainingJobAnalytics(estimator.latest_training_job.name ).dataframe()

								# extract kpis
								__lowerCAmelCase         =							list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"] )
								__lowerCAmelCase         =							list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"] )
								# get train time from SageMaker job, this includes starting, preprocessing, stopping
								__lowerCAmelCase         =							(
								    Session().describe_training_job(estimator.latest_training_job.name ).get("TrainingTimeInSeconds"      ,				999_999 )
								)

								# assert kpis
								assert train_runtime <= self.results["train_runtime"]
								assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy )
								assert all(t <= self.results["eval_loss"] for t in eval_loss )

								# dump tests result into json file to share in PR
								with open(F"{estimator.latest_training_job.name}.json"      ,				"w" ) as outfile:
									json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}      ,				snake_case__ )








import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
					import torch

					from transformers.generation import DisjunctiveConstraint

@require_torch
class       a		(   unittest.TestCase					):







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							[[1, 2, 4], [1, 2, 3, 4]]
								__lowerCAmelCase         =							DisjunctiveConstraint(snake_case__ )
								self.assertTrue(isinstance(dc.token_ids      ,				snake_case__ ) )

								with self.assertRaises(snake_case__ ):
									DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]] ) )

								with self.assertRaises(snake_case__ ):
									DisjunctiveConstraint([torch.LongTensor([1, 2, 4] ), torch.LongTensor([1, 2, 3, 4, 5] )] )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							[[1, 2], [1, 2, 3, 4]]

								with self.assertRaises(snake_case__ ):
									DisjunctiveConstraint(snake_case__ )  # fails here







							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							[[1, 2, 3], [1, 2, 4]]

								__lowerCAmelCase         =							DisjunctiveConstraint(snake_case__ )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(1 )
								__lowerCAmelCase         =							stepped is True and completed is False and reset is False
								self.assertTrue(snake_case__ )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.current_seq == [1] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(2 )
								__lowerCAmelCase         =							stepped is True and completed is False and reset is False
								self.assertTrue(snake_case__ )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.current_seq == [1, 2] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(3 )
								__lowerCAmelCase         =							stepped is True and completed is True and reset is False
								self.assertTrue(snake_case__ )
								self.assertTrue(dc.completed )  # Completed!
								self.assertTrue(dc.current_seq == [1, 2, 3] )







							def       UpperCAmelCase__					(						self  :  Optional[Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							[[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

								__lowerCAmelCase         =							DisjunctiveConstraint(snake_case__ )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(1 )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.current_seq == [1] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(2 )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.current_seq == [1, 2] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(4 )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.current_seq == [1, 2, 4] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(5 )
								self.assertTrue(dc.completed )  # Completed!
								self.assertTrue(dc.current_seq == [1, 2, 4, 5] )

								dc.reset()

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(1 )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.remaining() == 3 )
								self.assertTrue(dc.current_seq == [1] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(2 )
								self.assertTrue(not dc.completed )
								self.assertTrue(dc.remaining() == 2 )
								self.assertTrue(dc.current_seq == [1, 2] )

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							dc.update(5 )
								self.assertTrue(dc.completed )  # Completed!
								self.assertTrue(dc.remaining() == 0 )
								self.assertTrue(dc.current_seq == [1, 2, 5] )






