
import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
UpperCamelCase_   	=	{"LayoutLMv2Config", "LayoutLMv3Config"}

@is_pipeline_test
class       a		(   unittest.TestCase					):
							lowercase_		:						str   							=  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							lowercase_		:						List[Any]   							=  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														lowercase_		:						Optional[int]   							=  {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														lowercase_		:						Optional[int]   							=  {
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}







							def       UpperCAmelCase__					(						self  :  List[Any]      ,				snake_case__  :  Dict      ,				snake_case__  :  int      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							ZeroShotClassificationPipeline(
								    model=snake_case__      ,				tokenizer=snake_case__      ,				candidate_labels=["polics", "health"] )
								return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]







							def       UpperCAmelCase__					(						self  :  Any      ,				snake_case__  :  List[str]      ,				snake_case__  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							classifier("Who are you voting for in 2020?"      ,				candidate_labels="politics" )
								self.assertEqual(snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ )], "scores": [ANY(snake_case__ )]} )

								# No kwarg
								__lowerCAmelCase         =							classifier("Who are you voting for in 2020?"      ,				["politics"] )
								self.assertEqual(snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ )], "scores": [ANY(snake_case__ )]} )

								__lowerCAmelCase         =							classifier("Who are you voting for in 2020?"      ,				candidate_labels=["politics"] )
								self.assertEqual(snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ )], "scores": [ANY(snake_case__ )]} )

								__lowerCAmelCase         =							classifier("Who are you voting for in 2020?"      ,				candidate_labels="politics, public health" )
								self.assertEqual(
								    snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ ), ANY(snake_case__ )], "scores": [ANY(snake_case__ ), ANY(snake_case__ )]} )
								self.assertAlmostEqual(sum(nested_simplify(outputs["scores"] ) )      ,				1.0 )

								__lowerCAmelCase         =							classifier("Who are you voting for in 2020?"      ,				candidate_labels=["politics", "public health"] )
								self.assertEqual(
								    snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ ), ANY(snake_case__ )], "scores": [ANY(snake_case__ ), ANY(snake_case__ )]} )
								self.assertAlmostEqual(sum(nested_simplify(outputs["scores"] ) )      ,				1.0 )

								__lowerCAmelCase         =							classifier(
								    "Who are you voting for in 2020?"      ,				candidate_labels="politics"      ,				hypothesis_template="This text is about {}" )
								self.assertEqual(snake_case__      ,				{"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ )], "scores": [ANY(snake_case__ )]} )

								# https://github.com/huggingface/transformers/issues/13846
								__lowerCAmelCase         =							classifier(["I am happy"]      ,				["positive", "negative"] )
								self.assertEqual(
								    snake_case__      ,				[
								        {"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ ), ANY(snake_case__ )], "scores": [ANY(snake_case__ ), ANY(snake_case__ )]}
								        for i in range(1 )
								    ]      ,				)
								__lowerCAmelCase         =							classifier(["I am happy", "I am sad"]      ,				["positive", "negative"] )
								self.assertEqual(
								    snake_case__      ,				[
								        {"sequence": ANY(snake_case__ ), "labels": [ANY(snake_case__ ), ANY(snake_case__ )], "scores": [ANY(snake_case__ ), ANY(snake_case__ )]}
								        for i in range(2 )
								    ]      ,				)

								with self.assertRaises(snake_case__ ):
									classifier(""      ,				candidate_labels="politics" )

								with self.assertRaises(snake_case__ ):
									classifier(snake_case__      ,				candidate_labels="politics" )

								with self.assertRaises(snake_case__ ):
									classifier("Who are you voting for in 2020?"      ,				candidate_labels="" )

								with self.assertRaises(snake_case__ ):
									classifier("Who are you voting for in 2020?"      ,				candidate_labels=snake_case__ )

								with self.assertRaises(snake_case__ ):
									classifier(
									    "Who are you voting for in 2020?"      ,				candidate_labels="politics"      ,				hypothesis_template="Not formatting template"      ,				)

								with self.assertRaises(snake_case__ ):
									classifier(
									    "Who are you voting for in 2020?"      ,				candidate_labels="politics"      ,				hypothesis_template=snake_case__      ,				)

								self.run_entailment_id(snake_case__ )







							def       UpperCAmelCase__					(						self  :  int      ,				snake_case__  :  Pipeline ):
								"""simple docstring"""







								__lowerCAmelCase         =							zero_shot_classifier.model.config
								__lowerCAmelCase         =							config.labelaid
								__lowerCAmelCase         =							zero_shot_classifier.entailment_id

								__lowerCAmelCase         =							{"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
								self.assertEqual(zero_shot_classifier.entailment_id      ,				-1 )

								__lowerCAmelCase         =							{"entailment": 0, "neutral": 1, "contradiction": 2}
								self.assertEqual(zero_shot_classifier.entailment_id      ,				0 )

								__lowerCAmelCase         =							{"ENTAIL": 0, "NON-ENTAIL": 1}
								self.assertEqual(zero_shot_classifier.entailment_id      ,				0 )

								__lowerCAmelCase         =							{"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
								self.assertEqual(zero_shot_classifier.entailment_id      ,				2 )

								__lowerCAmelCase         =							original_labelaid
								self.assertEqual(snake_case__      ,				zero_shot_classifier.entailment_id )







							@require_torch
							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							pipeline(
								    "zero-shot-classification"      ,				model="sshleifer/tiny-distilbert-base-cased-distilled-squad"      ,				framework="pt"      ,				)
								# There was a regression in 4.10 for this
								# Adding a test so we don't make the mistake again.
								# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
								zero_shot_classifier(
								    "Who are you voting for in 2020?" * 100      ,				candidate_labels=["politics", "public health", "science"] )







							@require_torch
							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							pipeline(
								    "zero-shot-classification"      ,				model="sshleifer/tiny-distilbert-base-cased-distilled-squad"      ,				framework="pt"      ,				)
								__lowerCAmelCase         =							zero_shot_classifier(
								    "Who are you voting for in 2020?"      ,				candidate_labels=["politics", "public health", "science"] )

								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": "Who are you voting for in 2020?",
								        "labels": ["science", "public health", "politics"],
								        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
								    }      ,				)







							@require_tf
							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							pipeline(
								    "zero-shot-classification"      ,				model="sshleifer/tiny-distilbert-base-cased-distilled-squad"      ,				framework="tf"      ,				)
								__lowerCAmelCase         =							zero_shot_classifier(
								    "Who are you voting for in 2020?"      ,				candidate_labels=["politics", "public health", "science"] )

								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": "Who are you voting for in 2020?",
								        "labels": ["science", "public health", "politics"],
								        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
								    }      ,				)







							@slow
							@require_torch
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							pipeline("zero-shot-classification"      ,				model="roberta-large-mnli"      ,				framework="pt" )
								__lowerCAmelCase         =							zero_shot_classifier(
								    "Who are you voting for in 2020?"      ,				candidate_labels=["politics", "public health", "science"] )

								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": "Who are you voting for in 2020?",
								        "labels": ["politics", "public health", "science"],
								        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
								    }      ,				)
								__lowerCAmelCase         =							zero_shot_classifier(
								    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
								    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
								    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
								    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
								    " machine translation tasks show these models to be superior in quality while being more parallelizable"
								    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
								    " English-to-German translation task, improving over the existing best results, including ensembles by"
								    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
								    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
								    " fraction of the training costs of the best models from the literature. We show that the Transformer"
								    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
								    " large and limited training data."      ,				candidate_labels=["machine learning", "statistics", "translation", "vision"]      ,				multi_label=snake_case__      ,				)
								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": (
								            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
								            " networks in an encoder-decoder configuration. The best performing models also connect the"
								            " encoder and decoder through an attention mechanism. We propose a new simple network"
								            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
								            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
								            " superior in quality while being more parallelizable and requiring significantly less time to"
								            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
								            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
								            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
								            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
								            " costs of the best models from the literature. We show that the Transformer generalizes well to"
								            " other tasks by applying it successfully to English constituency parsing both with large and"
								            " limited training data."
								        ),
								        "labels": ["translation", "machine learning", "vision", "statistics"],
								        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
								    }      ,				)







							@slow
							@require_tf
							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							pipeline("zero-shot-classification"      ,				model="roberta-large-mnli"      ,				framework="tf" )
								__lowerCAmelCase         =							zero_shot_classifier(
								    "Who are you voting for in 2020?"      ,				candidate_labels=["politics", "public health", "science"] )

								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": "Who are you voting for in 2020?",
								        "labels": ["politics", "public health", "science"],
								        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
								    }      ,				)
								__lowerCAmelCase         =							zero_shot_classifier(
								    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
								    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
								    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
								    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
								    " machine translation tasks show these models to be superior in quality while being more parallelizable"
								    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
								    " English-to-German translation task, improving over the existing best results, including ensembles by"
								    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
								    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
								    " fraction of the training costs of the best models from the literature. We show that the Transformer"
								    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
								    " large and limited training data."      ,				candidate_labels=["machine learning", "statistics", "translation", "vision"]      ,				multi_label=snake_case__      ,				)
								self.assertEqual(
								    nested_simplify(snake_case__ )      ,				{
								        "sequence": (
								            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
								            " networks in an encoder-decoder configuration. The best performing models also connect the"
								            " encoder and decoder through an attention mechanism. We propose a new simple network"
								            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
								            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
								            " superior in quality while being more parallelizable and requiring significantly less time to"
								            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
								            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
								            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
								            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
								            " costs of the best models from the literature. We show that the Transformer generalizes well to"
								            " other tasks by applying it successfully to English constituency parsing both with large and"
								            " limited training data."
								        ),
								        "labels": ["translation", "machine learning", "vision", "statistics"],
								        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
								    }      ,				)








import fire

from utils import calculate_rouge, save_json






def 		_UpperCAmelCase   (	UpperCamelCase:     Any			,							UpperCamelCase:     Union[str, Any]			,							UpperCamelCase:     List[Any]=None			,							**UpperCamelCase:     Optional[int]  ):






	"""simple docstring"""


	__lowerCAmelCase         =							[x.strip() for x in open(UpperCamelCase  ).readlines()]
	__lowerCAmelCase         =							[x.strip() for x in open(UpperCamelCase  ).readlines()][: len(UpperCamelCase  )]
	__lowerCAmelCase         =							calculate_rouge(UpperCamelCase			,							UpperCamelCase			,							**UpperCamelCase  )
	if save_path is not None:
		save_json(UpperCamelCase			,							UpperCamelCase			,							indent=UpperCamelCase  )
	return metrics  # these print nicely


if __name__ == "__main__":
					fire.Fire(calculate_rouge_path)






