


"""simple docstring"""






import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
	from sagemaker import Session, TrainingJobAnalytics
	from sagemaker.huggingface import HuggingFace


@pytest.mark.skipif(
    literal_eval(os.getenv("TEST_SAGEMAKER"       ,  "False"   )   ) is not True       ,  reason="Skipping test because should only be run when releasing minor transformers version"       ,  )
@pytest.mark.usefixtures("sm_env"   )
@parameterized_class(
    [
        {
            "framework": "pytorch",
            "script": "run_glue.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.g4dn.xlarge",
            "results": {"train_runtime": 650, "eval_accuracy": 0.6, "eval_loss": 0.9},
        },
        {
            "framework": "tensorflow",
            "script": "run_tf.py",
            "model_name_or_path": "distilbert-base-cased",
            "instance_type": "ml.g4dn.xlarge",
            "results": {"train_runtime": 600, "eval_accuracy": 0.3, "eval_loss": 0.9},
        },
    ]   )
class 			lowerCamelCase_		(			unittest.TestCase   ):





				"""simple docstring"""







				def   lowerCAmelCase__		(						self     ):
								if self.framework == "pytorch":
												subprocess.run(
												    f'''cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'''.split()       ,		encoding="utf-8"       ,		check=UpperCAmelCase__       ,		)
								assert hasattr(self       ,		"env"     )



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__=1     ):
								# creates estimator
								return HuggingFace(
								    entry_point=self.script       ,		source_dir=self.env.test_path       ,		role=self.env.role       ,		image_uri=self.env.image_uri       ,		base_job_name=f'''{self.env.base_job_name}-single'''       ,		instance_count=UpperCAmelCase__       ,		instance_type=self.instance_type       ,		debugger_hook_config=UpperCAmelCase__       ,		hyperparameters={**self.env.hyperparameters, "model_name_or_path": self.model_name_or_path}       ,		metric_definitions=self.env.metric_definitions       ,		py_version="py36"       ,		)



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__     ):
								TrainingJobAnalytics(UpperCAmelCase__     ).export_csv(f'''{self.env.test_path}/{job_name}_metrics.csv'''     )






				def   lowerCAmelCase__		(						self     ):
								# create estimator
								SCREAMING_SNAKE_CASE__     							=				self.create_estimator()

								# run training
								estimator.fit()

								# result dataframe
								SCREAMING_SNAKE_CASE__     							=				TrainingJobAnalytics(estimator.latest_training_job.name     ).dataframe()

								# extract kpis
								SCREAMING_SNAKE_CASE__     							=				list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]     )
								SCREAMING_SNAKE_CASE__     							=				list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]     )
								# get train time from SageMaker job, this includes starting, preprocessing, stopping
								SCREAMING_SNAKE_CASE__     							=				(
								    Session().describe_training_job(estimator.latest_training_job.name     ).get("TrainingTimeInSeconds"       ,		99_9999     )
								)

								# assert kpis
								assert train_runtime <= self.results["train_runtime"]
								assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy     )
								assert all(t <= self.results["eval_loss"] for t in eval_loss     )

								# dump tests result into json file to share in PR
								with open(f'''{estimator.latest_training_job.name}.json'''       ,		"w"     ) as outfile:
												json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}       ,		UpperCAmelCase__     )




"""simple docstring"""






import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


_lowerCamelCase     =  datasets.logging.get_logger(__name__)


_lowerCamelCase     =  '\\n@inproceedings{bleurt,\n  title={BLEURT: Learning Robust Metrics for Text Generation},\n  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},\n  booktitle={ACL},\n  year={2020},\n  url={https://arxiv.org/abs/2004.04696}\n}\n'

_lowerCamelCase     =  '\\nBLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)\nand then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune\nit for your specific application (the latter is expected to perform better).\n\nSee the project\'s README at https://github.com/google-research/bleurt#readme for more information.\n'

_lowerCamelCase     =  '\nBLEURT score.\n\nArgs:\n    `predictions` (list of str): prediction/candidate sentences\n    `references` (list of str): reference sentences\n    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.\n\nReturns:\n    \'scores\': List of scores.\nExamples:\n\n    >>> predictions = ["hello there", "general kenobi"]\n    >>> references = ["hello there", "general kenobi"]\n    >>> bleurt = datasets.load_metric("bleurt")\n    >>> results = bleurt.compute(predictions=predictions, references=references)\n    >>> print([round(v, 2) for v in results["scores"]])\n    [1.03, 1.04]\n'

_lowerCamelCase     =  {
    'bleurt-tiny-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip',
    'bleurt-tiny-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip',
    'bleurt-base-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip',
    'bleurt-base-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip',
    'bleurt-large-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip',
    'bleurt-large-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip',
    'BLEURT-20-D3': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip',
    'BLEURT-20-D6': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip',
    'BLEURT-20-D12': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip',
    'BLEURT-20': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip',
}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,  _KWARGS_DESCRIPTION   )
class 			lowerCamelCase_		(			datasets.Metric   ):





				"""simple docstring"""







				def   lowerCAmelCase__		(						self     ):
								return datasets.MetricInfo(
								    description=_DESCRIPTION       ,		citation=_CITATION       ,		homepage="https://github.com/google-research/bleurt"       ,		inputs_description=_KWARGS_DESCRIPTION       ,		features=datasets.Features(
								        {
								            "predictions": datasets.Value("string"       ,		id="sequence"     ),
								            "references": datasets.Value("string"       ,		id="sequence"     ),
								        }     )       ,		codebase_urls=["https://github.com/google-research/bleurt"]       ,		reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"]       ,		)



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__     ):
								# check that config name specifies a valid BLEURT model
								if self.config_name == "default":
												logger.warning(
												    "Using default BLEURT-Base checkpoint for sequence maximum length 128. "
												    "You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512')."     )
												SCREAMING_SNAKE_CASE__     							=				"bleurt-base-128"

								if self.config_name.lower() in CHECKPOINT_URLS:
												SCREAMING_SNAKE_CASE__     							=				self.config_name.lower()

								elif self.config_name.upper() in CHECKPOINT_URLS:
												SCREAMING_SNAKE_CASE__     							=				self.config_name.upper()

								else:
												raise KeyError(
												    f'''{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}'''     )

								# download the model checkpoint specified by self.config_name and set up the scorer
								SCREAMING_SNAKE_CASE__     							=				dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]     )
								SCREAMING_SNAKE_CASE__     							=				score.BleurtScorer(os.path.join(UpperCAmelCase__       ,		UpperCAmelCase__     )     )






				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__     ):
								SCREAMING_SNAKE_CASE__     							=				self.scorer.score(references=UpperCAmelCase__       ,		candidates=UpperCAmelCase__     )
								return {"scores": scores}
