

"""simple docstring"""

import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY






@is_pipeline_test
@require_torch_or_tf
class 			__a					(						unittest.TestCase       ):





       """simple docstring"""


       __UpperCamelCase					:      List[str]    =						MODEL_FOR_CAUSAL_LM_MAPPING
       __UpperCamelCase					:      List[str]    =						TF_MODEL_FOR_CAUSAL_LM_MAPPING






       @require_torch
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Any 			=				pipeline(task="text-generation"			,					model="sshleifer/tiny-ctrl"			,					framework="pt"   )
        # Using `do_sample=False` to force deterministic output
        lowerCAmelCase__   :				Any 			=				text_generator("This is a test"			,					do_sample=snake_case   )
        self.assertEqual(
            snake_case			,					[
                {
                    "generated_text": (
                        "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
                        " oscope. FiliFili@@"
                    )
                }
            ]			,					)

        lowerCAmelCase__   :				Tuple 			=				text_generator(["This is a test", "This is a second test"]   )
        self.assertEqual(
            snake_case			,					[
                [
                    {
                        "generated_text": (
                            "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
                            " oscope. FiliFili@@"
                        )
                    }
                ],
                [
                    {
                        "generated_text": (
                            "This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy"
                            " oscope. oscope. FiliFili@@"
                        )
                    }
                ],
            ]			,					)

        lowerCAmelCase__   :				Optional[int] 			=				text_generator("This is a test"			,					do_sample=snake_case			,					num_return_sequences=2			,					return_tensors=snake_case   )
        self.assertEqual(
            snake_case			,					[
                {"generated_token_ids": ANY(snake_case   )},
                {"generated_token_ids": ANY(snake_case   )},
            ]			,					)
        lowerCAmelCase__   :				List[str] 			=				text_generator.model.config.eos_token_id
        lowerCAmelCase__   :				List[Any] 			=				"<pad>"
        lowerCAmelCase__   :				Dict 			=				text_generator(
            ["This is a test", "This is a second test"]			,					do_sample=snake_case			,					num_return_sequences=2			,					batch_size=2			,					return_tensors=snake_case			,					)
        self.assertEqual(
            snake_case			,					[
                [
                    {"generated_token_ids": ANY(snake_case   )},
                    {"generated_token_ids": ANY(snake_case   )},
                ],
                [
                    {"generated_token_ids": ANY(snake_case   )},
                    {"generated_token_ids": ANY(snake_case   )},
                ],
            ]			,					)






       @require_tf
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Any 			=				pipeline(task="text-generation"			,					model="sshleifer/tiny-ctrl"			,					framework="tf"   )

        # Using `do_sample=False` to force deterministic output
        lowerCAmelCase__   :				List[Any] 			=				text_generator("This is a test"			,					do_sample=snake_case   )
        self.assertEqual(
            snake_case			,					[
                {
                    "generated_text": (
                        "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
                        " please,"
                    )
                }
            ]			,					)

        lowerCAmelCase__   :				Any 			=				text_generator(["This is a test", "This is a second test"]			,					do_sample=snake_case   )
        self.assertEqual(
            snake_case			,					[
                [
                    {
                        "generated_text": (
                            "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
                            " please,"
                        )
                    }
                ],
                [
                    {
                        "generated_text": (
                            "This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes"
                            " Cannes 閲閲Cannes Cannes Cannes 攵 please,"
                        )
                    }
                ],
            ]			,					)






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[int] 			=				TextGenerationPipeline(model=snake_case			,					tokenizer=snake_case   )
        return text_generator, ["This is a test", "Another test"]






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				int 			=				"Hello I believe in"
        lowerCAmelCase__   :				int 			=				pipeline("text-generation"			,					model="hf-internal-testing/tiny-random-gpt2"   )
        lowerCAmelCase__   :				List[str] 			=				text_generator(snake_case   )
        self.assertEqual(
            snake_case			,					[{"generated_text": "Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe"}]			,					)

        lowerCAmelCase__   :				List[Any] 			=				text_generator(snake_case			,					stop_sequence=" fe"   )
        self.assertEqual(snake_case			,					[{"generated_text": "Hello I believe in fe"}]   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Tuple 			=				text_generator.model
        lowerCAmelCase__   :				Optional[Any] 			=				text_generator.tokenizer

        lowerCAmelCase__   :				Any 			=				text_generator("This is a test"   )
        self.assertEqual(snake_case			,					[{"generated_text": ANY(snake_case   )}]   )
        self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"   )   )

        lowerCAmelCase__   :				str 			=				text_generator("This is a test"			,					return_full_text=snake_case   )
        self.assertEqual(snake_case			,					[{"generated_text": ANY(snake_case   )}]   )
        self.assertNotIn("This is a test"			,					outputs[0]["generated_text"]   )

        lowerCAmelCase__   :				Optional[int] 			=				pipeline(task="text-generation"			,					model=snake_case			,					tokenizer=snake_case			,					return_full_text=snake_case   )
        lowerCAmelCase__   :				Dict 			=				text_generator("This is a test"   )
        self.assertEqual(snake_case			,					[{"generated_text": ANY(snake_case   )}]   )
        self.assertNotIn("This is a test"			,					outputs[0]["generated_text"]   )

        lowerCAmelCase__   :				Any 			=				text_generator("This is a test"			,					return_full_text=snake_case   )
        self.assertEqual(snake_case			,					[{"generated_text": ANY(snake_case   )}]   )
        self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"   )   )

        lowerCAmelCase__   :				Union[str, Any] 			=				text_generator(["This is great !", "Something else"]			,					num_return_sequences=2			,					do_sample=snake_case   )
        self.assertEqual(
            snake_case			,					[
                [{"generated_text": ANY(snake_case   )}, {"generated_text": ANY(snake_case   )}],
                [{"generated_text": ANY(snake_case   )}, {"generated_text": ANY(snake_case   )}],
            ]			,					)

        if text_generator.tokenizer.pad_token is not None:
         lowerCAmelCase__   :				List[str] 			=				text_generator(
             ["This is great !", "Something else"]			,					num_return_sequences=2			,					batch_size=2			,					do_sample=snake_case   )
         self.assertEqual(
             snake_case			,					[
                 [{"generated_text": ANY(snake_case   )}, {"generated_text": ANY(snake_case   )}],
                 [{"generated_text": ANY(snake_case   )}, {"generated_text": ANY(snake_case   )}],
             ]			,					)

        with self.assertRaises(snake_case   ):
         lowerCAmelCase__   :				Any 			=				text_generator("test"			,					return_full_text=snake_case			,					return_text=snake_case   )
        with self.assertRaises(snake_case   ):
         lowerCAmelCase__   :				List[str] 			=				text_generator("test"			,					return_full_text=snake_case			,					return_tensors=snake_case   )
        with self.assertRaises(snake_case   ):
         lowerCAmelCase__   :				List[str] 			=				text_generator("test"			,					return_text=snake_case			,					return_tensors=snake_case   )

        # Empty prompt is slighly special
        # it requires BOS token to exist.
        # Special case for Pegasus which will always append EOS so will
        # work even without BOS.
        if (
            text_generator.tokenizer.bos_token_id is not None
            or "Pegasus" in tokenizer.__class__.__name__
            or "Git" in model.__class__.__name__
        ):
         lowerCAmelCase__   :				Dict 			=				text_generator(""   )
         self.assertEqual(snake_case			,					[{"generated_text": ANY(snake_case   )}]   )
        else:
         with self.assertRaises((ValueError, AssertionError)   ):
          lowerCAmelCase__   :				Optional[Any] 			=				text_generator(""   )

        if text_generator.framework == "tf":
         # TF generation does not support max_new_tokens, and it's impossible
         # to control long generation with only max_length without
         # fancy calculation, dismissing tests for now.
         return
        # We don't care about infinite range models.
        # They already work.
        # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
        lowerCAmelCase__   :				Any 			=				["RwkvForCausalLM", "XGLMForCausalLM", "GPTNeoXForCausalLM"]
        if (
            tokenizer.model_max_length < 10_000
            and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
        ):
         # Handling of large generations
         with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)   ):
          text_generator("This is a test" * 500			,					max_new_tokens=20   )

         lowerCAmelCase__   :				str 			=				text_generator("This is a test" * 500			,					handle_long_generation="hole"			,					max_new_tokens=20   )
         # Hole strategy cannot work
         with self.assertRaises(snake_case   ):
          text_generator(
              "This is a test" * 500			,					handle_long_generation="hole"			,					max_new_tokens=tokenizer.model_max_length + 10			,					)






       @require_torch
       @require_accelerate
       @require_torch_gpu
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        import torch

        # Classic `model_kwargs`
        lowerCAmelCase__   :				Optional[Any] 			=				pipeline(
            model="hf-internal-testing/tiny-random-bloom"			,					model_kwargs={"device_map": "auto", "torch_dtype": torch.bfloataa}			,					)
        self.assertEqual(pipe.model.device			,					torch.device(0   )   )
        self.assertEqual(pipe.model.lm_head.weight.dtype			,					torch.bfloataa   )
        lowerCAmelCase__   :				Optional[int] 			=				pipe("This is a test"   )
        self.assertEqual(
            snake_case			,					[
                {
                    "generated_text": (
                        "This is a test test test test test test test test test test test test test test test test"
                        " test"
                    )
                }
            ]			,					)

        # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
        lowerCAmelCase__   :				List[str] 			=				pipeline(model="hf-internal-testing/tiny-random-bloom"			,					device_map="auto"			,					torch_dtype=torch.bfloataa   )
        self.assertEqual(pipe.model.device			,					torch.device(0   )   )
        self.assertEqual(pipe.model.lm_head.weight.dtype			,					torch.bfloataa   )
        lowerCAmelCase__   :				Any 			=				pipe("This is a test"   )
        self.assertEqual(
            snake_case			,					[
                {
                    "generated_text": (
                        "This is a test test test test test test test test test test test test test test test test"
                        " test"
                    )
                }
            ]			,					)

        # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
        lowerCAmelCase__   :				int 			=				pipeline(model="hf-internal-testing/tiny-random-bloom"			,					device_map="auto"   )
        self.assertEqual(pipe.model.device			,					torch.device(0   )   )
        self.assertEqual(pipe.model.lm_head.weight.dtype			,					torch.floataa   )
        lowerCAmelCase__   :				Dict 			=				pipe("This is a test"   )
        self.assertEqual(
            snake_case			,					[
                {
                    "generated_text": (
                        "This is a test test test test test test test test test test test test test test test test"
                        " test"
                    )
                }
            ]			,					)






       @require_torch
       @require_torch_gpu
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        import torch

        lowerCAmelCase__   :				Optional[Any] 			=				pipeline(model="hf-internal-testing/tiny-random-bloom"			,					device=0			,					torch_dtype=torch.floataa   )
        pipe("This is a test"   )






       @require_torch
       @require_accelerate
       @require_torch_gpu
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        import torch

        lowerCAmelCase__   :				Optional[Any] 			=				pipeline(model="hf-internal-testing/tiny-random-bloom"			,					device_map="auto"			,					torch_dtype=torch.floataa   )
        pipe("This is a test"			,					do_sample=snake_case			,					top_p=0.5   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				"Hello world"
        lowerCAmelCase__   :				Optional[Any] 			=				pipeline("text-generation"			,					model="hf-internal-testing/tiny-random-gpt2"   )
        if text_generator.model.framework == "tf":
         lowerCAmelCase__   :				Tuple 			=				logging.get_logger("transformers.generation.tf_utils"   )
        else:
         lowerCAmelCase__   :				Union[str, Any] 			=				logging.get_logger("transformers.generation.utils"   )
        lowerCAmelCase__   :				Optional[int] 			=				"Both `max_new_tokens`"  # The beggining of the message to be checked in this test

        # Both are set by the user -> log warning
        with CaptureLogger(snake_case   ) as cl:
         lowerCAmelCase__   :				Union[str, Any] 			=				text_generator(snake_case			,					max_length=10			,					max_new_tokens=1   )
        self.assertIn(snake_case			,					cl.out   )

        # The user only sets one -> no warning
        with CaptureLogger(snake_case   ) as cl:
         lowerCAmelCase__   :				List[str] 			=				text_generator(snake_case			,					max_new_tokens=1   )
        self.assertNotIn(snake_case			,					cl.out   )

        with CaptureLogger(snake_case   ) as cl:
         lowerCAmelCase__   :				Tuple 			=				text_generator(snake_case			,					max_length=10   )
        self.assertNotIn(snake_case			,					cl.out   )









"""simple docstring"""

import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			__a					(						__magic_name__       ,	unittest.TestCase       ):





       """simple docstring"""


       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizer
       __UpperCamelCase					:      Optional[Any]    =						LxmertTokenizerFast
       __UpperCamelCase					:      str    =						True
       __UpperCamelCase					:      Any    =						True






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        super().setUp()

        lowerCAmelCase__   :				Tuple 			=				[
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        lowerCAmelCase__   :				Dict 			=				os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES["vocab_file"]   )
        with open(self.vocab_file			,					"w"			,					encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				"UNwant\u00E9d,running"
        lowerCAmelCase__   :				Union[str, Any] 			=				"unwanted, running"
        return input_text, output_text






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				str 			=				self.tokenizer_class(self.vocab_file   )

        lowerCAmelCase__   :				List[Any] 			=				tokenizer.tokenize("UNwant\u00E9d,running"   )
        self.assertListEqual(snake_case			,					["un", "##want", "##ed", ",", "runn", "##ing"]   )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case   )			,					[7, 4, 5, 10, 8, 9]   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        if not self.test_rust_tokenizer:
         return

        lowerCAmelCase__   :				int 			=				self.get_tokenizer()
        lowerCAmelCase__   :				List[str] 			=				self.get_rust_tokenizer()

        lowerCAmelCase__   :				Optional[Any] 			=				"I was born in 92000, and this is falsé."

        lowerCAmelCase__   :				int 			=				tokenizer.tokenize(snake_case   )
        lowerCAmelCase__   :				int 			=				rust_tokenizer.tokenize(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				str 			=				tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        lowerCAmelCase__   :				List[str] 			=				rust_tokenizer.encode(snake_case			,					add_special_tokens=snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )

        lowerCAmelCase__   :				Tuple 			=				self.get_rust_tokenizer()
        lowerCAmelCase__   :				Tuple 			=				tokenizer.encode(snake_case   )
        lowerCAmelCase__   :				List[Any] 			=				rust_tokenizer.encode(snake_case   )
        self.assertListEqual(snake_case			,					snake_case   )






