





from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    flip_channel_order,
    get_resize_output_image_size,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, is_vision_available, logging


if is_vision_available():
		import PIL

if is_torch_available():
		import torch


UpperCAmelCase_  							=				logging.get_logger(__name__)






class 					lowercase__ (			__lowerCamelCase       ):

			'''simple docstring'''


			a		:							Optional[Any]     	=	["pixel_values"]




			def __init__(					self,	__magic_name__ = True,	__magic_name__ = None,	__magic_name__ = PILImageResampling.BILINEAR,	__magic_name__ = True,	__magic_name__ = 1 / 255,	__magic_name__ = True,	__magic_name__ = None,	__magic_name__ = True,	**__magic_name__,	)				->   None:






							"""simple docstring"""
							super().__init__(**__magic_name__							)
							UpperCamelCase__							:       List[Any]      =   size if size is not None else {'''shortest_edge''': 224}
							UpperCamelCase__							:       str      =   get_size_dict(__magic_name__,	default_to_square=__magic_name__							)
							UpperCamelCase__							:       List[str]      =   crop_size if crop_size is not None else {'''height''': 256, '''width''': 256}
							UpperCamelCase__							:       List[str]      =   get_size_dict(__magic_name__,	param_name='''crop_size'''							)

							UpperCamelCase__							:       Union[str, Any]      =   do_resize
							UpperCamelCase__							:       Union[str, Any]      =   size
							UpperCamelCase__							:       Tuple      =   resample
							UpperCamelCase__							:       Union[str, Any]      =   do_rescale
							UpperCamelCase__							:       Optional[Any]      =   rescale_factor
							UpperCamelCase__							:       Optional[int]      =   do_center_crop
							UpperCamelCase__							:       Dict      =   crop_size
							UpperCamelCase__							:       Union[str, Any]      =   do_flip_channel_order




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__ = PIL.Image.BILINEAR,	__magic_name__ = None,	**__magic_name__,	)				->   np.ndarray:






							"""simple docstring"""
							UpperCamelCase__							:       Optional[Any]      =   get_size_dict(__magic_name__,	default_to_square=__magic_name__							)
							if "shortest_edge" not in size:
											raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}"							)
							UpperCamelCase__							:       Optional[Any]      =   get_resize_output_image_size(__magic_name__,	size=size['''shortest_edge'''],	default_to_square=__magic_name__							)
							return resize(__magic_name__,	size=__magic_name__,	resample=__magic_name__,	data_format=__magic_name__,	**__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__ = None,	**__magic_name__,	)				->   np.ndarray:






							"""simple docstring"""
							UpperCamelCase__							:       int      =   get_size_dict(__magic_name__							)
							if "height" not in size or "width" not in size:
											raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}"							)
							return center_crop(__magic_name__,	size=(size['''height'''], size['''width''']),	data_format=__magic_name__,	**__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__ = None,	**__magic_name__,	)				->   str:






							"""simple docstring"""
							return rescale(__magic_name__,	scale=__magic_name__,	data_format=__magic_name__,	**__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   np.ndarray:






							"""simple docstring"""
							return flip_channel_order(__magic_name__,	data_format=__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = None,	__magic_name__ = ChannelDimension.FIRST,	**__magic_name__,	)				->   PIL.Image.Image:






							"""simple docstring"""
							UpperCamelCase__							:       Union[str, Any]      =   do_resize if do_resize is not None else self.do_resize
							UpperCamelCase__							:       Optional[Any]      =   resample if resample is not None else self.resample
							UpperCamelCase__							:       List[str]      =   do_rescale if do_rescale is not None else self.do_rescale
							UpperCamelCase__							:       Tuple      =   rescale_factor if rescale_factor is not None else self.rescale_factor
							UpperCamelCase__							:       int      =   do_center_crop if do_center_crop is not None else self.do_center_crop
							UpperCamelCase__							:       Union[str, Any]      =   (
							    do_flip_channel_order if do_flip_channel_order is not None else self.do_flip_channel_order
							)

							UpperCamelCase__							:       List[Any]      =   size if size is not None else self.size
							UpperCamelCase__							:       Optional[Any]      =   get_size_dict(__magic_name__,	default_to_square=__magic_name__							)
							UpperCamelCase__							:       List[str]      =   crop_size if crop_size is not None else self.crop_size
							UpperCamelCase__							:       Any      =   get_size_dict(__magic_name__,	param_name='''crop_size'''							)

							UpperCamelCase__							:       List[Any]      =   make_list_of_images(__magic_name__							)

							if not valid_images(__magic_name__							):
											raise ValueError(
											    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
											    '''torch.Tensor, tf.Tensor or jax.ndarray.'''							)

							if do_resize and size is None:
											raise ValueError('''Size must be specified if do_resize is True.'''							)

							if do_rescale and rescale_factor is None:
											raise ValueError('''Rescale factor must be specified if do_rescale is True.'''							)

							if do_center_crop and crop_size is None:
											raise ValueError('''Crop size must be specified if do_center_crop is True.'''							)

							# All transformations expect numpy arrays.
							UpperCamelCase__							:       List[Any]      =   [to_numpy_array(__magic_name__							) for image in images]

							if do_resize:
											UpperCamelCase__							:       Union[str, Any]      =   [self.resize(image=__magic_name__,	size=__magic_name__,	resample=__magic_name__							) for image in images]

							if do_center_crop:
											UpperCamelCase__							:       Optional[int]      =   [self.center_crop(image=__magic_name__,	size=__magic_name__							) for image in images]

							if do_rescale:
											UpperCamelCase__							:       List[Any]      =   [self.rescale(image=__magic_name__,	scale=__magic_name__							) for image in images]

							# the pretrained checkpoints assume images are BGR, not RGB
							if do_flip_channel_order:
											UpperCamelCase__							:       List[Any]      =   [self.flip_channel_order(image=__magic_name__							) for image in images]

							UpperCamelCase__							:       List[str]      =   [to_channel_dimension_format(__magic_name__,	__magic_name__							) for image in images]

							UpperCamelCase__							:       Optional[int]      =   {'''pixel_values''': images}
							return BatchFeature(data=__magic_name__,	tensor_type=__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       str      =   outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
											if len(__magic_name__							) != len(__magic_name__							):
															raise ValueError(
															    '''Make sure that you pass in as many target sizes as the batch dimension of the logits'''							)

											if is_torch_tensor(__magic_name__							):
															UpperCamelCase__							:       Optional[int]      =   target_sizes.numpy()

											UpperCamelCase__							:       int      =   []

											for idx in range(len(__magic_name__							)							):
															UpperCamelCase__							:       Optional[int]      =   torch.nn.functional.interpolate(
															    logits[idx].unsqueeze(dim=0							),	size=target_sizes[idx],	mode='''bilinear''',	align_corners=__magic_name__							)
															UpperCamelCase__							:       Tuple      =   resized_logits[0].argmax(dim=0							)
															semantic_segmentation.append(__magic_name__							)
							else:
											UpperCamelCase__							:       Optional[Any]      =   logits.argmax(dim=1							)
											UpperCamelCase__							:       List[str]      =   [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]							)]

							return semantic_segmentation








import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase_  							=				logging.get_logger(__name__)

UpperCAmelCase_  							=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_  							=				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_  							=				{
    'Salesforce/codegen-350M-mono': 2048,
}






class 					lowercase__ (			__lowerCamelCase       ):

   '''simple docstring'''


   a		:							Union[str, Any]     	=	VOCAB_FILES_NAMES
   a		:							List[str]     	=	PRETRAINED_VOCAB_FILES_MAP
   a		:							Optional[int]     	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a		:							List[Any]     	=	["input_ids", "attention_mask"]
   a		:							Any     	=	CodeGenTokenizer




   def __init__(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__=False,	**__magic_name__,	)				->   Optional[int]:






       """simple docstring"""
       super().__init__(
           __magic_name__,	__magic_name__,	tokenizer_file=__magic_name__,	unk_token=__magic_name__,	bos_token=__magic_name__,	eos_token=__magic_name__,	add_prefix_space=__magic_name__,	**__magic_name__,	)

       if kwargs.pop('''add_bos_token''',	__magic_name__							):
           UpperCamelCase__							:       List[Any]      =   kwargs.pop('''name_or_path''',	''''''							)
           raise ValueError(
               '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
               '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
               f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
               f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
               '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
               ''' so that the fast tokenizer works correctly.'''							)

       UpperCamelCase__							:       Dict      =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',	__magic_name__							) != add_prefix_space:
           UpperCamelCase__							:       List[str]      =   getattr(__magic_name__,	pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:       int      =   add_prefix_space
           UpperCamelCase__							:       List[str]      =   pre_tok_class(**__magic_name__							)

       UpperCamelCase__							:       List[Any]      =   add_prefix_space




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   kwargs.get('''is_split_into_words''',	__magic_name__							)
       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   kwargs.get('''is_split_into_words''',	__magic_name__							)

       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Tuple[str]:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   self._tokenizer.model.save(__magic_name__,	name=__magic_name__							)
       return tuple(__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = False,	__magic_name__ = None,	__magic_name__ = None,	**__magic_name__,	)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   super().decode(
           token_ids=__magic_name__,	skip_special_tokens=__magic_name__,	clean_up_tokenization_spaces=__magic_name__,	**__magic_name__,	)

       if truncate_before_pattern is not None and len(__magic_name__							) > 0:
           UpperCamelCase__							:       str      =   self.truncate(__magic_name__,	__magic_name__							)

       return decoded_text




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__							)				->   List[str]:






       """simple docstring"""
       def find_re(__magic_name__,	__magic_name__,	__magic_name__							):
           UpperCamelCase__							:       Dict      =   pattern.search(__magic_name__,	__magic_name__							)
           return m.start() if m else -1

       UpperCamelCase__							:       Union[str, Any]      =   [re.compile(__magic_name__,	re.MULTILINE							) for pattern in truncate_before_pattern]

       UpperCamelCase__							:       List[Any]      =   list(re.finditer('''^print''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Tuple      =   completion[: prints[1].start()]

       UpperCamelCase__							:       Optional[int]      =   list(re.finditer('''^def''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Optional[Any]      =   completion[: defs[1].start()]

       UpperCamelCase__							:       str      =   0

       UpperCamelCase__							:       Tuple      =   [
           pos for pos in [find_re(__magic_name__,	__magic_name__,	__magic_name__							) for terminal in terminals] if pos != -1
       ]

       if len(__magic_name__							) > 0:
           return completion[: min(__magic_name__							)]
       else:
           return completion

