
"""simple docstring"""
import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
      import torch

if is_tf_available():
      import tensorflow as tf
@require_vision
@require_torchvision
class 			UpperCamelCase      (							unittest.TestCase  ):
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Dict							    = tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE   :				Any							    = SamImageProcessor()
       _SCREAMING_SNAKE_CASE   :				List[str]							    = SamProcessor(snake_case__				)
       processor.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       return AutoProcessor.from_pretrained(self.tmpdirname  ,     **snake_case__				).image_processor
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       shutil.rmtree(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[str]							    = [np.random.randint(255  ,     size=(3, 30, 400)  ,     dtype=np.uinta				)]

       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = [Image.fromarray(np.moveaxis(snake_case__  ,     0  ,     -1				)				) for x in image_inputs]

       return image_inputs
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = SamProcessor(image_processor=self.get_image_processor()				)
       processor.save_pretrained(self.tmpdirname				)

       _SCREAMING_SNAKE_CASE   :				Dict							    = self.get_image_processor(do_normalize=snake_case__  ,     padding_value=1.0				)

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = SamProcessor.from_pretrained(self.tmpdirname  ,     do_normalize=snake_case__  ,     padding_value=1.0				)

       self.assertEqual(processor.image_processor.to_json_string()  ,     image_processor_add_kwargs.to_json_string()				)
       self.assertIsInstance(processor.image_processor  ,     snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				Any							    = SamProcessor(image_processor=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = image_processor(snake_case__  ,     return_tensors="np"				)
       _SCREAMING_SNAKE_CASE   :				Any							    = processor(images=snake_case__  ,     return_tensors="np"				)

       input_feat_extract.pop("original_sizes"				)  # pop original_sizes as it is popped in the processor
       input_feat_extract.pop("reshaped_input_sizes"				)  # pop original_sizes as it is popped in the processor

       for key in input_feat_extract.keys():
         self.assertAlmostEqual(input_feat_extract[key].sum()  ,     input_processor[key].sum()  ,     delta=1E-2				)




     @require_torch
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Tuple							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				Dict							    = SamProcessor(image_processor=snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = [torch.ones((1, 3, 5, 5)				)]

       _SCREAMING_SNAKE_CASE   :				int							    = [[1764, 2646]]

       _SCREAMING_SNAKE_CASE   :				Dict							    = [[683, 1024]]
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = processor.post_process_masks(snake_case__  ,     snake_case__  ,     snake_case__				)
       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       _SCREAMING_SNAKE_CASE   :				str							    = processor.post_process_masks(
           snake_case__  ,     torch.tensor(snake_case__				)  ,     torch.tensor(snake_case__				)				)
       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       # should also work with np
       _SCREAMING_SNAKE_CASE   :				Any							    = [np.ones((1, 3, 5, 5)				)]
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = processor.post_process_masks(snake_case__  ,     np.array(snake_case__				)  ,     np.array(snake_case__				)				)

       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = [[1, 0], [0, 1]]
       with self.assertRaises(snake_case__				):
         _SCREAMING_SNAKE_CASE   :				int							    = processor.post_process_masks(snake_case__  ,     np.array(snake_case__				)  ,     np.array(snake_case__				)				)
@require_vision
@require_tf
class 			UpperCamelCase      (							unittest.TestCase  ):
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Tuple							    = tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = SamImageProcessor()
       _SCREAMING_SNAKE_CASE   :				str							    = SamProcessor(snake_case__				)
       processor.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       return AutoProcessor.from_pretrained(self.tmpdirname  ,     **snake_case__				).image_processor
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       shutil.rmtree(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[str]							    = [np.random.randint(255  ,     size=(3, 30, 400)  ,     dtype=np.uinta				)]

       _SCREAMING_SNAKE_CASE   :				int							    = [Image.fromarray(np.moveaxis(snake_case__  ,     0  ,     -1				)				) for x in image_inputs]

       return image_inputs
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = SamProcessor(image_processor=self.get_image_processor()				)
       processor.save_pretrained(self.tmpdirname				)

       _SCREAMING_SNAKE_CASE   :				int							    = self.get_image_processor(do_normalize=snake_case__  ,     padding_value=1.0				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = SamProcessor.from_pretrained(self.tmpdirname  ,     do_normalize=snake_case__  ,     padding_value=1.0				)

       self.assertEqual(processor.image_processor.to_json_string()  ,     image_processor_add_kwargs.to_json_string()				)
       self.assertIsInstance(processor.image_processor  ,     snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				str							    = SamProcessor(image_processor=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE   :				str							    = image_processor(snake_case__  ,     return_tensors="np"				)
       _SCREAMING_SNAKE_CASE   :				str							    = processor(images=snake_case__  ,     return_tensors="np"				)

       input_feat_extract.pop("original_sizes"				)  # pop original_sizes as it is popped in the processor
       input_feat_extract.pop("reshaped_input_sizes"				)  # pop reshaped_input_sizes as it is popped in the processor

       for key in input_feat_extract.keys():
         self.assertAlmostEqual(input_feat_extract[key].sum()  ,     input_processor[key].sum()  ,     delta=1E-2				)




     @require_tf
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Dict							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = SamProcessor(image_processor=snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = [tf.ones((1, 3, 5, 5)				)]

       _SCREAMING_SNAKE_CASE   :				List[str]							    = [[1764, 2646]]

       _SCREAMING_SNAKE_CASE   :				Tuple							    = [[683, 1024]]
       _SCREAMING_SNAKE_CASE   :				int							    = processor.post_process_masks(snake_case__  ,     snake_case__  ,     snake_case__  ,     return_tensors="tf"				)
       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = processor.post_process_masks(
           snake_case__  ,     tf.convert_to_tensor(snake_case__				)  ,     tf.convert_to_tensor(snake_case__				)  ,     return_tensors="tf"  ,     )
       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       # should also work with np
       _SCREAMING_SNAKE_CASE   :				str							    = [np.ones((1, 3, 5, 5)				)]
       _SCREAMING_SNAKE_CASE   :				List[str]							    = processor.post_process_masks(
           snake_case__  ,     np.array(snake_case__				)  ,     np.array(snake_case__				)  ,     return_tensors="tf"				)

       self.assertEqual(masks[0].shape  ,     (1, 3, 1764, 2646)				)

       _SCREAMING_SNAKE_CASE   :				int							    = [[1, 0], [0, 1]]
       with self.assertRaises(tf.errors.InvalidArgumentError				):
         _SCREAMING_SNAKE_CASE   :				str							    = processor.post_process_masks(
             snake_case__  ,     np.array(snake_case__				)  ,     np.array(snake_case__				)  ,     return_tensors="tf"				)
@require_vision
@require_torchvision
class 			UpperCamelCase      (							unittest.TestCase  ):
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE   :				Tuple							    = SamImageProcessor()
       _SCREAMING_SNAKE_CASE   :				Any							    = SamProcessor(snake_case__				)
       processor.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       return AutoProcessor.from_pretrained(self.tmpdirname  ,     **snake_case__				).image_processor
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       shutil.rmtree(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[str]							    = [np.random.randint(255  ,     size=(3, 30, 400)  ,     dtype=np.uinta				)]

       _SCREAMING_SNAKE_CASE   :				Dict							    = [Image.fromarray(np.moveaxis(snake_case__  ,     0  ,     -1				)				) for x in image_inputs]

       return image_inputs
     @is_pt_tf_cross_test
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				List[str]							    = SamProcessor(image_processor=snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = np.random.randint(0  ,     2  ,     size=(1, 3, 5, 5)				).astype(np.floataa				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = [tf.convert_to_tensor(snake_case__				)]
       _SCREAMING_SNAKE_CASE   :				Any							    = [torch.tensor(snake_case__				)]

       _SCREAMING_SNAKE_CASE   :				Tuple							    = [[1764, 2646]]

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = [[683, 1024]]
       _SCREAMING_SNAKE_CASE   :				Tuple							    = processor.post_process_masks(
           snake_case__  ,     snake_case__  ,     snake_case__  ,     return_tensors="tf"				)
       _SCREAMING_SNAKE_CASE   :				Any							    = processor.post_process_masks(
           snake_case__  ,     snake_case__  ,     snake_case__  ,     return_tensors="pt"				)

       self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()				)				)




     @is_pt_tf_cross_test
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.get_image_processor()

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = SamProcessor(image_processor=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE   :				int							    = image_processor(snake_case__  ,     return_tensors="pt"				)["pixel_values"].numpy()
       _SCREAMING_SNAKE_CASE   :				str							    = processor(images=snake_case__  ,     return_tensors="pt"				)["pixel_values"].numpy()

       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = image_processor(snake_case__  ,     return_tensors="tf"				)["pixel_values"].numpy()
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = processor(images=snake_case__  ,     return_tensors="tf"				)["pixel_values"].numpy()

       self.assertTrue(np.allclose(snake_case__  ,     snake_case__				)				)
       self.assertTrue(np.allclose(snake_case__  ,     snake_case__				)				)
       self.assertTrue(np.allclose(snake_case__  ,     snake_case__				)				)





"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be trained."""}  )
     A__      =			field(
         default="""./"""      ,       metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path of training dataset."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for training."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for evaluation."""}  )
     A__      =			field(default=0.1      ,       metadata={"""help""": """Value of weight decay."""}  )
     A__      =			field(
         default=10000      ,       metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""}  )
     A__      =			field(default=2e-4      ,       metadata={"""help""": """Learning rate fo training."""}  )
     A__      =			field(default="""cosine"""      ,       metadata={"""help""": """Learning rate."""}  )
     A__      =			field(
         default=750      ,       metadata={"""help""": """Number of warmup steps in the learning rate schedule."""}  )
     A__      =			field(
         default=16      ,       metadata={"""help""": """Number of gradient accumulation steps."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""}  )
     A__      =			field(default=50000      ,       metadata={"""help""": """Maximum number of training steps."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Sequence lengths used for training."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Training seed."""}  )
     A__      =			field(
         default=1024      ,       metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """States path if the training should continue from a checkpoint folder."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True the data is pretokenized."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size used for evaluation."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Length of sequences to be evaluated."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Sample from the language model's output distribution."""}  )
     A__      =			field(default=0.2      ,       metadata={"""help""": """Sampling temperature used for generation."""}  )
     A__      =			field(default=256      ,       metadata={"""help""": """Maximum number of newly generated tokens."""}  )
     A__      =			field(default=0      ,       metadata={"""help""": """Top-k parameter used for generation."""}  )
     A__      =			field(default=0.95      ,       metadata={"""help""": """Top-p parameter used for nucleus sampling."""}  )
     A__      =			field(default=10      ,       metadata={"""help""": """Number of generations to run in parallel."""}  )
     A__      =			field(
         default=200      ,       metadata={"""help""": """Number of completions to generate for each sample."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""eval_results.json"""      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""0"""      ,       metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""}  )
     A__      =			field(
         default=-1      ,       metadata={
             """help""": (
                 """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
                 """ number corresponds to which GPU device id to run on."""
             )
         }      ,       )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={
             """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
         }      ,       )
     A__      =			field(
         default="""transformersbook/codeparrot"""      ,       metadata={"""help""": """Folder or name of dataset to process."""}  )
     A__      =			field(
         default="""codeparrot-clean"""      ,       metadata={"""help""": """Folder to save processed processed dataset."""}  )
     A__      =			field(
         default=100000      ,       metadata={"""help""": """Number of files to save per JSON output file."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(
         default=1000      ,       metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=100      ,       metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.25      ,       metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""}  )
     A__      =			field(
         default=1.5      ,       metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.7      ,       metadata={"""help""": """Probability for filtering config, test and uncommon files."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True, near-duplicate samples are removed."""}  )
     A__      =			field(
         default=0.85      ,       metadata={"""help""": """Jaccard threshold for near-duplicate samples."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2"""      ,       metadata={"""help""": """Base tokenizer to build new tokenizer from."""}  )
     A__      =			field(
         default="""transformersbook/codeparrot-train"""      ,       metadata={"""help""": """Dataset to train tokenizer on."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(default=200000      ,       metadata={"""help""": """Number of examples to train tokenizer on."""}  )
     A__      =			field(
         default=32768      ,       metadata={"""help""": """Number of examples to train the tokenizer on."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of new tokenizer."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path to the dataset to pretokenize."""}  )
     A__      =			field(
         default="""tokenized-codeparrot-train"""      ,       metadata={"""help""": """Repo name of the pretokenized data."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2-large"""      ,       metadata={"""help""": """Configuration to use for model initialization."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Tokenizer attached to model."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of the created model."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )



