



'''simple docstring'''





import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
SCREAMING_SNAKE_CASE__      					=     {"LayoutLMv2Config", "LayoutLMv3Config"}


@is_pipeline_test
class 				snake_case (unittest.TestCase   ):
       lowerCAmelCase__  :List[Any]						   =					MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
       lowerCAmelCase__  :str						   =					TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

       if model_mapping is not None:
              lowerCAmelCase__  :List[str]						   =					{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
       if tf_model_mapping is not None:
              lowerCAmelCase__  :Any						   =					{
                  config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
              }



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						ZeroShotClassificationPipeline(
               model=UpperCAmelCase_     ,tokenizer=UpperCAmelCase_     ,candidate_labels=["polics", "health"]      )
           return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Union[str, Any]:
           lowercase__		       =						classifier("Who are you voting for in 2020?"     ,candidate_labels="politics"      )
           self.assertEqual(UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      )]}      )

           # No kwarg
           lowercase__		       =						classifier("Who are you voting for in 2020?"     ,["politics"]      )
           self.assertEqual(UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      )]}      )

           lowercase__		       =						classifier("Who are you voting for in 2020?"     ,candidate_labels=["politics"]      )
           self.assertEqual(UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      )]}      )

           lowercase__		       =						classifier("Who are you voting for in 2020?"     ,candidate_labels="politics, public health"      )
           self.assertEqual(
               UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )]}      )
           self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]      )      )     ,1.0      )

           lowercase__		       =						classifier("Who are you voting for in 2020?"     ,candidate_labels=["politics", "public health"]      )
           self.assertEqual(
               UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )]}      )
           self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]      )      )     ,1.0      )

           lowercase__		       =						classifier(
               "Who are you voting for in 2020?"     ,candidate_labels="politics"     ,hypothesis_template="This text is about {}"      )
           self.assertEqual(UpperCAmelCase_     ,{"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      )]}      )

           # https://github.com/huggingface/transformers/issues/13846
           lowercase__		       =						classifier(["I am happy"]     ,["positive", "negative"]      )
           self.assertEqual(
               UpperCAmelCase_     ,[
                   {"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )]}
                   for i in range(1      )
               ]     ,)
           lowercase__		       =						classifier(["I am happy", "I am sad"]     ,["positive", "negative"]      )
           self.assertEqual(
               UpperCAmelCase_     ,[
                   {"sequence": ANY(UpperCAmelCase_      ), "labels": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )], "scores": [ANY(UpperCAmelCase_      ), ANY(UpperCAmelCase_      )]}
                   for i in range(2      )
               ]     ,)

           with self.assertRaises(UpperCAmelCase_      ):
               classifier(""     ,candidate_labels="politics"      )

           with self.assertRaises(UpperCAmelCase_      ):
               classifier(UpperCAmelCase_     ,candidate_labels="politics"      )

           with self.assertRaises(UpperCAmelCase_      ):
               classifier("Who are you voting for in 2020?"     ,candidate_labels=""      )

           with self.assertRaises(UpperCAmelCase_      ):
               classifier("Who are you voting for in 2020?"     ,candidate_labels=UpperCAmelCase_      )

           with self.assertRaises(UpperCAmelCase_      ):
               classifier(
                   "Who are you voting for in 2020?"     ,candidate_labels="politics"     ,hypothesis_template="Not formatting template"     ,)

           with self.assertRaises(UpperCAmelCase_      ):
               classifier(
                   "Who are you voting for in 2020?"     ,candidate_labels="politics"     ,hypothesis_template=UpperCAmelCase_     ,)

           self.run_entailment_id(UpperCAmelCase_      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							Optional[Any]:
           lowercase__		       =						zero_shot_classifier.model.config
           lowercase__		       =						config.labelaid
           lowercase__		       =						zero_shot_classifier.entailment_id

           lowercase__		       =						{"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
           self.assertEqual(zero_shot_classifier.entailment_id     ,-1      )

           lowercase__		       =						{"entailment": 0, "neutral": 1, "contradiction": 2}
           self.assertEqual(zero_shot_classifier.entailment_id     ,0      )

           lowercase__		       =						{"ENTAIL": 0, "NON-ENTAIL": 1}
           self.assertEqual(zero_shot_classifier.entailment_id     ,0      )

           lowercase__		       =						{"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
           self.assertEqual(zero_shot_classifier.entailment_id     ,2      )

           lowercase__		       =						original_labelaid
           self.assertEqual(UpperCAmelCase_     ,zero_shot_classifier.entailment_id      )



       @require_torch
       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						pipeline(
               "zero-shot-classification"     ,model="sshleifer/tiny-distilbert-base-cased-distilled-squad"     ,framework="pt"     ,)
           # There was a regression in 4.10 for this
           # Adding a test so we don't make the mistake again.
           # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
           zero_shot_classifier(
               "Who are you voting for in 2020?" * 100     ,candidate_labels=["politics", "public health", "science"]      )



       @require_torch
       def 							_a      (				self      )  ->							Tuple:
           lowercase__		       =						pipeline(
               "zero-shot-classification"     ,model="sshleifer/tiny-distilbert-base-cased-distilled-squad"     ,framework="pt"     ,)
           lowercase__		       =						zero_shot_classifier(
               "Who are you voting for in 2020?"     ,candidate_labels=["politics", "public health", "science"]      )

           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["science", "public health", "politics"],
                   "scores": [0.3_33, 0.3_33, 0.3_33],
               }     ,)



       @require_tf
       def 							_a      (				self      )  ->							Optional[Any]:
           lowercase__		       =						pipeline(
               "zero-shot-classification"     ,model="sshleifer/tiny-distilbert-base-cased-distilled-squad"     ,framework="tf"     ,)
           lowercase__		       =						zero_shot_classifier(
               "Who are you voting for in 2020?"     ,candidate_labels=["politics", "public health", "science"]      )

           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["science", "public health", "politics"],
                   "scores": [0.3_33, 0.3_33, 0.3_33],
               }     ,)



       @slow
       @require_torch
       def 							_a      (				self      )  ->							str:
           lowercase__		       =						pipeline("zero-shot-classification"     ,model="roberta-large-mnli"     ,framework="pt"      )
           lowercase__		       =						zero_shot_classifier(
               "Who are you voting for in 2020?"     ,candidate_labels=["politics", "public health", "science"]      )

           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["politics", "public health", "science"],
                   "scores": [0.9_76, 0.0_15, 0.0_09],
               }     ,)
           lowercase__		       =						zero_shot_classifier(
               "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
               " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
               " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
               " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
               " machine translation tasks show these models to be superior in quality while being more parallelizable"
               " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
               " English-to-German translation task, improving over the existing best results, including ensembles by"
               " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
               " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
               " fraction of the training costs of the best models from the literature. We show that the Transformer"
               " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
               " large and limited training data."     ,candidate_labels=["machine learning", "statistics", "translation", "vision"]     ,multi_label=UpperCAmelCase_     ,)
           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": (
                       "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                       " networks in an encoder-decoder configuration. The best performing models also connect the"
                       " encoder and decoder through an attention mechanism. We propose a new simple network"
                       " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                       " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                       " superior in quality while being more parallelizable and requiring significantly less time to"
                       " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                       " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                       " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                       " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                       " costs of the best models from the literature. We show that the Transformer generalizes well to"
                       " other tasks by applying it successfully to English constituency parsing both with large and"
                       " limited training data."
                   ),
                   "labels": ["translation", "machine learning", "vision", "statistics"],
                   "scores": [0.8_17, 0.7_13, 0.0_18, 0.0_18],
               }     ,)







       @slow
       @require_tf
       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						pipeline("zero-shot-classification"     ,model="roberta-large-mnli"     ,framework="tf"      )
           lowercase__		       =						zero_shot_classifier(
               "Who are you voting for in 2020?"     ,candidate_labels=["politics", "public health", "science"]      )

           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["politics", "public health", "science"],
                   "scores": [0.9_76, 0.0_15, 0.0_09],
               }     ,)
           lowercase__		       =						zero_shot_classifier(
               "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
               " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
               " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
               " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
               " machine translation tasks show these models to be superior in quality while being more parallelizable"
               " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
               " English-to-German translation task, improving over the existing best results, including ensembles by"
               " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
               " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
               " fraction of the training costs of the best models from the literature. We show that the Transformer"
               " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
               " large and limited training data."     ,candidate_labels=["machine learning", "statistics", "translation", "vision"]     ,multi_label=UpperCAmelCase_     ,)
           self.assertEqual(
               nested_simplify(UpperCAmelCase_      )     ,{
                   "sequence": (
                       "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                       " networks in an encoder-decoder configuration. The best performing models also connect the"
                       " encoder and decoder through an attention mechanism. We propose a new simple network"
                       " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                       " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                       " superior in quality while being more parallelizable and requiring significantly less time to"
                       " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                       " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                       " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                       " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                       " costs of the best models from the literature. We show that the Transformer generalizes well to"
                       " other tasks by applying it successfully to English constituency parsing both with large and"
                       " limited training data."
                   ),
                   "labels": ["translation", "machine learning", "vision", "statistics"],
                   "scores": [0.8_17, 0.7_13, 0.0_18, 0.0_18],
               }     ,)









'''simple docstring'''





from typing import List, Optional

from tokenizers import ByteLevelBPETokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot_small import BlenderbotSmallTokenizer


SCREAMING_SNAKE_CASE__      					=     logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      					=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

SCREAMING_SNAKE_CASE__      					=     {
    "vocab_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
    },
    "merges_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
    },
    "tokenizer_config_file": {
        "facebook/blenderbot_small-90M": (
            "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"
        )
    },
}

SCREAMING_SNAKE_CASE__      					=     {
    "facebook/blenderbot_small-90M": 512,
}


class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Union[str, Any]						   =					VOCAB_FILES_NAMES
       lowerCAmelCase__  :Any						   =					PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__  :str						   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__  :List[Any]						   =					BlenderbotSmallTokenizer



       def __init__(				self     ,UpperCAmelCase_=None     ,UpperCAmelCase_=None     ,UpperCAmelCase_="<|endoftext|>"     ,UpperCAmelCase_="<|endoftext|>"     ,UpperCAmelCase_="<|endoftext|>"     ,UpperCAmelCase_=False     ,UpperCAmelCase_=True     ,**UpperCAmelCase_     ,)  ->							Optional[Any]:
           super().__init__(
               ByteLevelBPETokenizer(
                   vocab=UpperCAmelCase_     ,merges=UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_     ,trim_offsets=UpperCAmelCase_     ,)     ,bos_token=UpperCAmelCase_     ,eos_token=UpperCAmelCase_     ,unk_token=UpperCAmelCase_     ,**UpperCAmelCase_     ,)
           lowercase__		       =						add_prefix_space



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_=None      )  ->							Optional[int]:
           lowercase__		       =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
           if token_ids_a is None:
               return output

           return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None      )  ->							List[int]:
           lowercase__		       =						[self.sep_token_id]
           lowercase__		       =						[self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep      ) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]




