




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
lowercase__							:	str											=		{"LayoutLMv2Config", "LayoutLMv3Config"}
@is_pipeline_test
class 						UpperCAmelCase  (							unittest.TestCase     ):







						'''simple docstring'''





						lowerCAmelCase_							=  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
						lowerCAmelCase_							=  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

						if model_mapping is not None:
												lowerCAmelCase_							=  {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
						if tf_model_mapping is not None:
												lowerCAmelCase_							=  {
												    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
												}






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	Any			,      __lowercase				:	str			,      __lowercase				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	ZeroShotClassificationPipeline(
							    model=__lowercase			,      tokenizer=__lowercase			,      candidate_labels=["polics", "health"]		)
							return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	Tuple			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							snake_case_						       =	classifier("Who are you voting for in 2020?"			,      candidate_labels="politics"		)
							self.assertEqual(__lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		)], "scores": [ANY(__lowercase		)]}		)

							# No kwarg
							snake_case_						       =	classifier("Who are you voting for in 2020?"			,      ["politics"]		)
							self.assertEqual(__lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		)], "scores": [ANY(__lowercase		)]}		)

							snake_case_						       =	classifier("Who are you voting for in 2020?"			,      candidate_labels=["politics"]		)
							self.assertEqual(__lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		)], "scores": [ANY(__lowercase		)]}		)

							snake_case_						       =	classifier("Who are you voting for in 2020?"			,      candidate_labels="politics, public health"		)
							self.assertEqual(
							    __lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		), ANY(__lowercase		)], "scores": [ANY(__lowercase		), ANY(__lowercase		)]}		)
							self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]		)		)			,      1.0		)

							snake_case_						       =	classifier("Who are you voting for in 2020?"			,      candidate_labels=["politics", "public health"]		)
							self.assertEqual(
							    __lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		), ANY(__lowercase		)], "scores": [ANY(__lowercase		), ANY(__lowercase		)]}		)
							self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]		)		)			,      1.0		)

							snake_case_						       =	classifier(
							    "Who are you voting for in 2020?"			,      candidate_labels="politics"			,      hypothesis_template="This text is about {}"		)
							self.assertEqual(__lowercase			,      {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		)], "scores": [ANY(__lowercase		)]}		)

							# https://github.com/huggingface/transformers/issues/13846
							snake_case_						       =	classifier(["I am happy"]			,      ["positive", "negative"]		)
							self.assertEqual(
							    __lowercase			,      [
							        {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		), ANY(__lowercase		)], "scores": [ANY(__lowercase		), ANY(__lowercase		)]}
							        for i in range(1		)
							    ]			,      )
							snake_case_						       =	classifier(["I am happy", "I am sad"]			,      ["positive", "negative"]		)
							self.assertEqual(
							    __lowercase			,      [
							        {"sequence": ANY(__lowercase		), "labels": [ANY(__lowercase		), ANY(__lowercase		)], "scores": [ANY(__lowercase		), ANY(__lowercase		)]}
							        for i in range(2		)
							    ]			,      )

							with self.assertRaises(__lowercase		):
								classifier(""			,      candidate_labels="politics"		)

							with self.assertRaises(__lowercase		):
								classifier(__lowercase			,      candidate_labels="politics"		)

							with self.assertRaises(__lowercase		):
								classifier("Who are you voting for in 2020?"			,      candidate_labels=""		)

							with self.assertRaises(__lowercase		):
								classifier("Who are you voting for in 2020?"			,      candidate_labels=__lowercase		)

							with self.assertRaises(__lowercase		):
								classifier(
								    "Who are you voting for in 2020?"			,      candidate_labels="politics"			,      hypothesis_template="Not formatting template"			,      )

							with self.assertRaises(__lowercase		):
								classifier(
								    "Who are you voting for in 2020?"			,      candidate_labels="politics"			,      hypothesis_template=__lowercase			,      )

							self.run_entailment_id(__lowercase		)






						def 			snake_case__		(		self				:	Dict			,      __lowercase				:	Pipeline		):






							"""simple docstring"""






							snake_case_						       =	zero_shot_classifier.model.config
							snake_case_						       =	config.labelaid
							snake_case_						       =	zero_shot_classifier.entailment_id

							snake_case_						       =	{"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
							self.assertEqual(zero_shot_classifier.entailment_id			,      -1		)

							snake_case_						       =	{"entailment": 0, "neutral": 1, "contradiction": 2}
							self.assertEqual(zero_shot_classifier.entailment_id			,      0		)

							snake_case_						       =	{"ENTAIL": 0, "NON-ENTAIL": 1}
							self.assertEqual(zero_shot_classifier.entailment_id			,      0		)

							snake_case_						       =	{"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
							self.assertEqual(zero_shot_classifier.entailment_id			,      2		)

							snake_case_						       =	original_labelaid
							self.assertEqual(__lowercase			,      zero_shot_classifier.entailment_id		)






						@require_torch
						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							snake_case_						       =	pipeline(
							    "zero-shot-classification"			,      model="sshleifer/tiny-distilbert-base-cased-distilled-squad"			,      framework="pt"			,      )
							# There was a regression in 4.10 for this
							# Adding a test so we don't make the mistake again.
							# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
							zero_shot_classifier(
							    "Who are you voting for in 2020?" * 1_00			,      candidate_labels=["politics", "public health", "science"]		)






						@require_torch
						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							snake_case_						       =	pipeline(
							    "zero-shot-classification"			,      model="sshleifer/tiny-distilbert-base-cased-distilled-squad"			,      framework="pt"			,      )
							snake_case_						       =	zero_shot_classifier(
							    "Who are you voting for in 2020?"			,      candidate_labels=["politics", "public health", "science"]		)

							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": "Who are you voting for in 2020?",
							        "labels": ["science", "public health", "politics"],
							        "scores": [0.333, 0.333, 0.333],
							    }			,      )






						@require_tf
						def 			snake_case__		(		self				:	Dict		):






							"""simple docstring"""






							snake_case_						       =	pipeline(
							    "zero-shot-classification"			,      model="sshleifer/tiny-distilbert-base-cased-distilled-squad"			,      framework="tf"			,      )
							snake_case_						       =	zero_shot_classifier(
							    "Who are you voting for in 2020?"			,      candidate_labels=["politics", "public health", "science"]		)

							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": "Who are you voting for in 2020?",
							        "labels": ["science", "public health", "politics"],
							        "scores": [0.333, 0.333, 0.333],
							    }			,      )






						@slow
						@require_torch
						def 			snake_case__		(		self				:	List[str]		):






							"""simple docstring"""






							snake_case_						       =	pipeline("zero-shot-classification"			,      model="roberta-large-mnli"			,      framework="pt"		)
							snake_case_						       =	zero_shot_classifier(
							    "Who are you voting for in 2020?"			,      candidate_labels=["politics", "public health", "science"]		)

							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": "Who are you voting for in 2020?",
							        "labels": ["politics", "public health", "science"],
							        "scores": [0.976, 0.015, 0.009],
							    }			,      )
							snake_case_						       =	zero_shot_classifier(
							    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
							    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
							    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
							    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
							    " machine translation tasks show these models to be superior in quality while being more parallelizable"
							    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
							    " English-to-German translation task, improving over the existing best results, including ensembles by"
							    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
							    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
							    " fraction of the training costs of the best models from the literature. We show that the Transformer"
							    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
							    " large and limited training data."			,      candidate_labels=["machine learning", "statistics", "translation", "vision"]			,      multi_label=__lowercase			,      )
							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": (
							            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
							            " networks in an encoder-decoder configuration. The best performing models also connect the"
							            " encoder and decoder through an attention mechanism. We propose a new simple network"
							            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
							            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
							            " superior in quality while being more parallelizable and requiring significantly less time to"
							            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
							            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
							            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
							            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
							            " costs of the best models from the literature. We show that the Transformer generalizes well to"
							            " other tasks by applying it successfully to English constituency parsing both with large and"
							            " limited training data."
							        ),
							        "labels": ["translation", "machine learning", "vision", "statistics"],
							        "scores": [0.817, 0.713, 0.018, 0.018],
							    }			,      )






						@slow
						@require_tf
						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	pipeline("zero-shot-classification"			,      model="roberta-large-mnli"			,      framework="tf"		)
							snake_case_						       =	zero_shot_classifier(
							    "Who are you voting for in 2020?"			,      candidate_labels=["politics", "public health", "science"]		)

							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": "Who are you voting for in 2020?",
							        "labels": ["politics", "public health", "science"],
							        "scores": [0.976, 0.015, 0.009],
							    }			,      )
							snake_case_						       =	zero_shot_classifier(
							    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
							    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
							    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
							    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
							    " machine translation tasks show these models to be superior in quality while being more parallelizable"
							    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
							    " English-to-German translation task, improving over the existing best results, including ensembles by"
							    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
							    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
							    " fraction of the training costs of the best models from the literature. We show that the Transformer"
							    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
							    " large and limited training data."			,      candidate_labels=["machine learning", "statistics", "translation", "vision"]			,      multi_label=__lowercase			,      )
							self.assertEqual(
							    nested_simplify(__lowercase		)			,      {
							        "sequence": (
							            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
							            " networks in an encoder-decoder configuration. The best performing models also connect the"
							            " encoder and decoder through an attention mechanism. We propose a new simple network"
							            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
							            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
							            " superior in quality while being more parallelizable and requiring significantly less time to"
							            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
							            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
							            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
							            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
							            " costs of the best models from the literature. We show that the Transformer generalizes well to"
							            " other tasks by applying it successfully to English constituency parsing both with large and"
							            " limited training data."
							        ),
							        "labels": ["translation", "machine learning", "vision", "statistics"],
							        "scores": [0.817, 0.713, 0.018, 0.018],
							    }			,      )






import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
		from sagemaker import Session, TrainingJobAnalytics
		from sagemaker.huggingface import HuggingFace
@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''			,  '''False'''     )     ) is not True			,  reason='''Skipping test because should only be run when releasing minor transformers version'''			,  )
@pytest.mark.usefixtures('''sm_env'''     )
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue_model_parallelism.py''',
            '''model_name_or_path''': '''roberta-large''',
            '''instance_type''': '''ml.p3dn.24xlarge''',
            '''results''': {'''train_runtime''': 1600, '''eval_accuracy''': 0.3, '''eval_loss''': 1.2},
        },
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''roberta-large''',
            '''instance_type''': '''ml.p3dn.24xlarge''',
            '''results''': {'''train_runtime''': 1600, '''eval_accuracy''': 0.3, '''eval_loss''': 1.2},
        },
    ]     )
class 						UpperCAmelCase  (							unittest.TestCase     ):







						'''simple docstring'''





						def 			snake_case__		(		self				:	int		):






							"""simple docstring"""






							if self.framework == "pytorch":
								subprocess.run(
								    f"cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py".split()			,      encoding="utf-8"			,      check=__lowercase			,      )
							assert hasattr(self			,      "env"		)






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	Dict		):






							"""simple docstring"""






							snake_case_						       =	{
							    "enabled": True,
							    "processes_per_host": 8,
							}
							snake_case_						       =	{
							    "enabled": True,
							    "parameters": {
							        "microbatches": 4,
							        "placement_strategy": "spread",
							        "pipeline": "interleaved",
							        "optimize": "speed",
							        "partitions": 4,
							        "ddp": True,
							    },
							}

							snake_case_						       =	{"smdistributed": {"modelparallel": smp_options}, "mpi": mpi_options}

							snake_case_						       =	"trainer" if self.script == "run_glue.py" else "smtrainer"
							# creates estimator
							return HuggingFace(
							    entry_point=self.script			,      source_dir=self.env.test_path			,      role=self.env.role			,      image_uri=self.env.image_uri			,      base_job_name=f"{self.env.base_job_name}-{instance_count}-smp-{name_extension}"			,      instance_count=__lowercase			,      instance_type=self.instance_type			,      debugger_hook_config=__lowercase			,      hyperparameters={
							        **self.env.hyperparameters,
							        "model_name_or_path": self.model_name_or_path,
							        "max_steps": 5_00,
							    }			,      metric_definitions=self.env.metric_definitions			,      distribution=__lowercase			,      py_version="py36"			,      )






						def 			snake_case__		(		self				:	Tuple			,      __lowercase				:	Dict		):






							"""simple docstring"""






							TrainingJobAnalytics(__lowercase		).export_csv(f"{self.env.test_path}/{job_name}_metrics.csv"		)






						@parameterized.expand([(1,)]		)
						def 			snake_case__		(		self				:	Any			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	self.create_estimator(__lowercase		)

							# run training
							estimator.fit()

							# result dataframe
							snake_case_						       =	TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

							# extract kpis
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_accuracy"]["value"]		)
							snake_case_						       =	list(result_metrics_df[result_metrics_df.metric_name == "eval_loss"]["value"]		)
							# get train time from SageMaker job, this includes starting, preprocessing, stopping
							snake_case_						       =	(
							    Session().describe_training_job(estimator.latest_training_job.name		).get("TrainingTimeInSeconds"			,      99_99_99		)
							)

							# assert kpis
							assert train_runtime <= self.results["train_runtime"]
							assert all(t >= self.results["eval_accuracy"] for t in eval_accuracy		)
							assert all(t <= self.results["eval_loss"] for t in eval_loss		)

							# dump tests result into json file to share in PR
							with open(f"{estimator.latest_training_job.name}.json"			,      "w"		) as outfile:
								json.dump({"train_time": train_runtime, "eval_accuracy": eval_accuracy, "eval_loss": eval_loss}			,      __lowercase		)
