



import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY



@is_pipeline_test
@require_torch_or_tf
class 							lowerCAmelCase_						(    unittest.TestCase				):
       UpperCAmelCase__							:   Optional[int]  	=    MODEL_FOR_CAUSAL_LM_MAPPING
       UpperCAmelCase__							:   Union[str, Any]  	=    TF_MODEL_FOR_CAUSAL_LM_MAPPING
       @require_torch
       def    snake_case_			(     self			)	->						int:
          UpperCamelCase       :    Any		      =      pipeline(task='text-generation',     model='sshleifer/tiny-ctrl',     framework='pt'			)
          # Using `do_sample=False` to force deterministic output
          UpperCamelCase       :    Union[str, Any]		      =      text_generator('This is a test',     do_sample=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {
                      'generated_text': (
                          'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
                          ' oscope. FiliFili@@'
                      )
                  }
              ],     )

          UpperCamelCase       :    Any		      =      text_generator(['This is a test', 'This is a second test']			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  [
                      {
                          'generated_text': (
                              'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
                              ' oscope. FiliFili@@'
                          )
                      }
                  ],
                  [
                      {
                          'generated_text': (
                              'This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'
                              ' oscope. oscope. FiliFili@@'
                          )
                      }
                  ],
              ],     )

          UpperCamelCase       :    Optional[int]		      =      text_generator('This is a test',     do_sample=SCREAMING_SNAKE_CASE_,     num_return_sequences=2,     return_tensors=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
                  {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
              ],     )
          UpperCamelCase       :    Union[str, Any]		      =      text_generator.model.config.eos_token_id
          UpperCamelCase       :    Union[str, Any]		      =      '<pad>'
          UpperCamelCase       :    str		      =      text_generator(
              ['This is a test', 'This is a second test'],     do_sample=SCREAMING_SNAKE_CASE_,     num_return_sequences=2,     batch_size=2,     return_tensors=SCREAMING_SNAKE_CASE_,     )
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  [
                      {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
                      {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
                  ],
                  [
                      {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
                      {'generated_token_ids': ANY(SCREAMING_SNAKE_CASE_			)},
                  ],
              ],     )
       @require_tf
       def    snake_case_			(     self			)	->						str:
          UpperCamelCase       :    Tuple		      =      pipeline(task='text-generation',     model='sshleifer/tiny-ctrl',     framework='tf'			)

          # Using `do_sample=False` to force deterministic output
          UpperCamelCase       :    Optional[int]		      =      text_generator('This is a test',     do_sample=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {
                      'generated_text': (
                          'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
                          ' please,'
                      )
                  }
              ],     )

          UpperCamelCase       :    Dict		      =      text_generator(['This is a test', 'This is a second test'],     do_sample=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  [
                      {
                          'generated_text': (
                              'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
                              ' please,'
                          )
                      }
                  ],
                  [
                      {
                          'generated_text': (
                              'This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'
                              ' Cannes 閲閲Cannes Cannes Cannes 攵 please,'
                          )
                      }
                  ],
              ],     )
       def    snake_case_			(     self,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)	->						int:
          UpperCamelCase       :    List[Any]		      =      TextGenerationPipeline(model=SCREAMING_SNAKE_CASE_,     tokenizer=SCREAMING_SNAKE_CASE_			)
          return text_generator, ["This is a test", "Another test"]
       def    snake_case_			(     self			)	->						List[Any]:
          UpperCamelCase       :    Tuple		      =      'Hello I believe in'
          UpperCamelCase       :    Optional[int]		      =      pipeline('text-generation',     model='hf-internal-testing/tiny-random-gpt2'			)
          UpperCamelCase       :    List[str]		      =      text_generator(SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [{'generated_text': 'Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'}],     )

          UpperCamelCase       :    Optional[Any]		      =      text_generator(SCREAMING_SNAKE_CASE_,     stop_sequence=' fe'			)
          self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': 'Hello I believe in fe'}]			)
       def    snake_case_			(     self,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)	->						int:
          UpperCamelCase       :    Optional[int]		      =      text_generator.model
          UpperCamelCase       :    List[Any]		      =      text_generator.tokenizer

          UpperCamelCase       :    Union[str, Any]		      =      text_generator('This is a test'			)
          self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}]			)
          self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'			)			)

          UpperCamelCase       :    Optional[int]		      =      text_generator('This is a test',     return_full_text=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}]			)
          self.assertNotIn('This is a test',     outputs[0]['generated_text']			)

          UpperCamelCase       :    Tuple		      =      pipeline(task='text-generation',     model=SCREAMING_SNAKE_CASE_,     tokenizer=SCREAMING_SNAKE_CASE_,     return_full_text=SCREAMING_SNAKE_CASE_			)
          UpperCamelCase       :    int		      =      text_generator('This is a test'			)
          self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}]			)
          self.assertNotIn('This is a test',     outputs[0]['generated_text']			)

          UpperCamelCase       :    Dict		      =      text_generator('This is a test',     return_full_text=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}]			)
          self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'			)			)

          UpperCamelCase       :    List[Any]		      =      text_generator(['This is great !', 'Something else'],     num_return_sequences=2,     do_sample=SCREAMING_SNAKE_CASE_			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}, {'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}],
                  [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}, {'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}],
              ],     )

          if text_generator.tokenizer.pad_token is not None:
             UpperCamelCase       :    Optional[Any]		      =      text_generator(
                 ['This is great !', 'Something else'],     num_return_sequences=2,     batch_size=2,     do_sample=SCREAMING_SNAKE_CASE_			)
             self.assertEqual(
                 SCREAMING_SNAKE_CASE_,     [
                     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}, {'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}],
                     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}, {'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}],
                 ],     )

          with self.assertRaises(SCREAMING_SNAKE_CASE_			):
             UpperCamelCase       :    Any		      =      text_generator('test',     return_full_text=SCREAMING_SNAKE_CASE_,     return_text=SCREAMING_SNAKE_CASE_			)
          with self.assertRaises(SCREAMING_SNAKE_CASE_			):
             UpperCamelCase       :    Any		      =      text_generator('test',     return_full_text=SCREAMING_SNAKE_CASE_,     return_tensors=SCREAMING_SNAKE_CASE_			)
          with self.assertRaises(SCREAMING_SNAKE_CASE_			):
             UpperCamelCase       :    int		      =      text_generator('test',     return_text=SCREAMING_SNAKE_CASE_,     return_tensors=SCREAMING_SNAKE_CASE_			)

          # Empty prompt is slighly special
          # it requires BOS token to exist.
          # Special case for Pegasus which will always append EOS so will
          # work even without BOS.
          if (
              text_generator.tokenizer.bos_token_id is not None
              or "Pegasus" in tokenizer.__class__.__name__
              or "Git" in model.__class__.__name__
          ):
             UpperCamelCase       :    Dict		      =      text_generator(''			)
             self.assertEqual(SCREAMING_SNAKE_CASE_,     [{'generated_text': ANY(SCREAMING_SNAKE_CASE_			)}]			)
          else:
             with self.assertRaises((ValueError, AssertionError)			):
                UpperCamelCase       :    Optional[Any]		      =      text_generator(''			)

          if text_generator.framework == "tf":
             # TF generation does not support max_new_tokens, and it's impossible
             # to control long generation with only max_length without
             # fancy calculation, dismissing tests for now.
             return
          # We don't care about infinite range models.
          # They already work.
          # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
          UpperCamelCase       :    str		      =      ['RwkvForCausalLM', 'XGLMForCausalLM', 'GPTNeoXForCausalLM']
          if (
              tokenizer.model_max_length < 1_0000
              and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
          ):
             # Handling of large generations
             with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)			):
                text_generator('This is a test' * 500,     max_new_tokens=20			)

             UpperCamelCase       :    Dict		      =      text_generator('This is a test' * 500,     handle_long_generation='hole',     max_new_tokens=20			)
             # Hole strategy cannot work
             with self.assertRaises(SCREAMING_SNAKE_CASE_			):
                text_generator(
                    'This is a test' * 500,     handle_long_generation='hole',     max_new_tokens=tokenizer.model_max_length + 10,     )
       @require_torch
       @require_accelerate
       @require_torch_gpu
       def    snake_case_			(     self			)	->						Any:
          import torch

          # Classic `model_kwargs`
          UpperCamelCase       :    int		      =      pipeline(
              model='hf-internal-testing/tiny-random-bloom',     model_kwargs={'device_map': 'auto', 'torch_dtype': torch.bfloataa},     )
          self.assertEqual(pipe.model.device,     torch.device(0			)			)
          self.assertEqual(pipe.model.lm_head.weight.dtype,     torch.bfloataa			)
          UpperCamelCase       :    Any		      =      pipe('This is a test'			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {
                      'generated_text': (
                          'This is a test test test test test test test test test test test test test test test test'
                          ' test'
                      )
                  }
              ],     )

          # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
          UpperCamelCase       :    List[str]		      =      pipeline(model='hf-internal-testing/tiny-random-bloom',     device_map='auto',     torch_dtype=torch.bfloataa			)
          self.assertEqual(pipe.model.device,     torch.device(0			)			)
          self.assertEqual(pipe.model.lm_head.weight.dtype,     torch.bfloataa			)
          UpperCamelCase       :    Optional[Any]		      =      pipe('This is a test'			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {
                      'generated_text': (
                          'This is a test test test test test test test test test test test test test test test test'
                          ' test'
                      )
                  }
              ],     )

          # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
          UpperCamelCase       :    Dict		      =      pipeline(model='hf-internal-testing/tiny-random-bloom',     device_map='auto'			)
          self.assertEqual(pipe.model.device,     torch.device(0			)			)
          self.assertEqual(pipe.model.lm_head.weight.dtype,     torch.floataa			)
          UpperCamelCase       :    Optional[int]		      =      pipe('This is a test'			)
          self.assertEqual(
              SCREAMING_SNAKE_CASE_,     [
                  {
                      'generated_text': (
                          'This is a test test test test test test test test test test test test test test test test'
                          ' test'
                      )
                  }
              ],     )
       @require_torch
       @require_torch_gpu
       def    snake_case_			(     self			)	->						Union[str, Any]:
          import torch

          UpperCamelCase       :    Optional[Any]		      =      pipeline(model='hf-internal-testing/tiny-random-bloom',     device=0,     torch_dtype=torch.floataa			)
          pipe('This is a test'			)
       @require_torch
       @require_accelerate
       @require_torch_gpu
       def    snake_case_			(     self			)	->						int:
          import torch

          UpperCamelCase       :    Dict		      =      pipeline(model='hf-internal-testing/tiny-random-bloom',     device_map='auto',     torch_dtype=torch.floataa			)
          pipe('This is a test',     do_sample=SCREAMING_SNAKE_CASE_,     top_p=0.5			)






       def    snake_case_			(     self			)	->						Optional[int]:
          UpperCamelCase       :    Union[str, Any]		      =      'Hello world'
          UpperCamelCase       :    Optional[int]		      =      pipeline('text-generation',     model='hf-internal-testing/tiny-random-gpt2'			)
          if text_generator.model.framework == "tf":
             UpperCamelCase       :    Dict		      =      logging.get_logger('transformers.generation.tf_utils'			)
          else:
             UpperCamelCase       :    List[Any]		      =      logging.get_logger('transformers.generation.utils'			)
          UpperCamelCase       :    Dict		      =      'Both `max_new_tokens`'  # The beggining of the message to be checked in this test

          # Both are set by the user -> log warning
          with CaptureLogger(SCREAMING_SNAKE_CASE_			) as cl:
             UpperCamelCase       :    Tuple		      =      text_generator(SCREAMING_SNAKE_CASE_,     max_length=10,     max_new_tokens=1			)
          self.assertIn(SCREAMING_SNAKE_CASE_,     cl.out			)

          # The user only sets one -> no warning
          with CaptureLogger(SCREAMING_SNAKE_CASE_			) as cl:
             UpperCamelCase       :    Tuple		      =      text_generator(SCREAMING_SNAKE_CASE_,     max_new_tokens=1			)
          self.assertNotIn(SCREAMING_SNAKE_CASE_,     cl.out			)

          with CaptureLogger(SCREAMING_SNAKE_CASE_			) as cl:
             UpperCamelCase       :    List[str]		      =      text_generator(SCREAMING_SNAKE_CASE_,     max_length=10			)
          self.assertNotIn(SCREAMING_SNAKE_CASE_,     cl.out			)





'''simple docstring'''
import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import DonutImageProcessor


class 			SCREAMING_SNAKE_CASE__      (     unittest.TestCase):







				def __init__(       self						,				A_						,				A_=7						,				A_=3						,				A_=18						,				A_=30						,				A_=400						,				A_=True						,				A_=None						,				A_=True						,				A_=False						,				A_=True						,				A_=True						,				A_=[0.5, 0.5, 0.5]						,				A_=[0.5, 0.5, 0.5]						,				)->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     parent
											UpperCamelCase   			=     batch_size
											UpperCamelCase   			=     num_channels
											UpperCamelCase   			=     image_size
											UpperCamelCase   			=     min_resolution
											UpperCamelCase   			=     max_resolution
											UpperCamelCase   			=     do_resize
											UpperCamelCase   			=     size if size is not None else {'height': 18, 'width': 20}
											UpperCamelCase   			=     do_thumbnail
											UpperCamelCase   			=     do_align_axis
											UpperCamelCase   			=     do_pad
											UpperCamelCase   			=     do_normalize
											UpperCamelCase   			=     image_mean
											UpperCamelCase   			=     image_std







				def 						UpperCAmelCase_    (       self     )->		List[Any]:







											'''simple docstring'''






											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_thumbnail": self.do_thumbnail,
											    "do_align_long_axis": self.do_align_axis,
											    "do_pad": self.do_pad,
											    "do_normalize": self.do_normalize,
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											}


@require_torch
@require_vision
class 			SCREAMING_SNAKE_CASE__      (     snake_case_							,       unittest.TestCase):
				lowerCAmelCase_     			=							DonutImageProcessor if is_vision_available() else None







				def 						UpperCAmelCase_    (       self     )->		str:







											'''simple docstring'''






											UpperCamelCase   			=     DonutImageProcessingTester(self     )







				@property
				def 						UpperCAmelCase_    (       self     )->		str:







											'''simple docstring'''






											return self.image_processor_tester.prepare_image_processor_dict()







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     self.image_processing_class(**self.image_processor_dict     )
											self.assertTrue(hasattr(A_						,				'do_resize'     )     )
											self.assertTrue(hasattr(A_						,				'size'     )     )
											self.assertTrue(hasattr(A_						,				'do_thumbnail'     )     )
											self.assertTrue(hasattr(A_						,				'do_align_long_axis'     )     )
											self.assertTrue(hasattr(A_						,				'do_pad'     )     )
											self.assertTrue(hasattr(A_						,				'do_normalize'     )     )
											self.assertTrue(hasattr(A_						,				'image_mean'     )     )
											self.assertTrue(hasattr(A_						,				'image_std'     )     )







				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     self.image_processing_class.from_dict(self.image_processor_dict     )
											self.assertEqual(image_processor.size						,				{'height': 18, 'width': 20}     )

											UpperCamelCase   			=     self.image_processing_class.from_dict(self.image_processor_dict						,				size=42     )
											self.assertEqual(image_processor.size						,				{'height': 42, 'width': 42}     )

											# Previous config had dimensions in (width, height) order
											UpperCamelCase   			=     self.image_processing_class.from_dict(self.image_processor_dict						,				size=(42, 84)     )
											self.assertEqual(image_processor.size						,				{'height': 84, 'width': 42}     )







				def 						UpperCAmelCase_    (       self     )->		Tuple:







											'''simple docstring'''






											pass







				@is_flaky()
				def 						UpperCAmelCase_    (       self     )->		Any:







											'''simple docstring'''






											UpperCamelCase   			=     self.image_processing_class(**self.image_processor_dict     )
											# create random PIL images
											UpperCamelCase   			=     prepare_image_inputs(self.image_processor_tester						,				equal_resolution=A_     )
											for image in image_inputs:
																		self.assertIsInstance(A_						,				Image.Image     )

											# Test not batched input
											UpperCamelCase   			=     image_processing(image_inputs[0]						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)

											# Test batched
											UpperCamelCase   			=     image_processing(A_						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)







				@is_flaky()
				def 						UpperCAmelCase_    (       self     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     self.image_processing_class(**self.image_processor_dict     )
											# create random numpy tensors
											UpperCamelCase   			=     prepare_image_inputs(self.image_processor_tester						,				equal_resolution=A_						,				numpify=A_     )
											for image in image_inputs:
																		self.assertIsInstance(A_						,				np.ndarray     )

											# Test not batched input
											UpperCamelCase   			=     image_processing(image_inputs[0]						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)

											# Test batched
											UpperCamelCase   			=     image_processing(A_						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)







				@is_flaky()
				def 						UpperCAmelCase_    (       self     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     self.image_processing_class(**self.image_processor_dict     )
											# create random PyTorch tensors
											UpperCamelCase   			=     prepare_image_inputs(self.image_processor_tester						,				equal_resolution=A_						,				torchify=A_     )
											for image in image_inputs:
																		self.assertIsInstance(A_						,				torch.Tensor     )

											# Test not batched input
											UpperCamelCase   			=     image_processing(image_inputs[0]						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)

											# Test batched
											UpperCamelCase   			=     image_processing(A_						,				return_tensors='pt'     ).pixel_values
											self.assertEqual(
											    encoded_images.shape						,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['height'],
											        self.image_processor_tester.size['width'],
											    )						,				)




