





from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
		import torch

if is_tf_available():
		import tensorflow as tf






class 					lowercase__ (			__snake_case       ):

			'''simple docstring'''


			a		:							int     	=	["image_processor"]
			a		:							List[Any]     	=	"SamImageProcessor"




			def __init__(					self,	__magic_name__							)				->   Union[str, Any]:






							"""simple docstring"""
							super().__init__(__UpperCamelCase							)
							UpperCamelCase__							:       int      =   self.image_processor
							UpperCamelCase__							:       List[Any]      =   -10
							UpperCamelCase__							:       List[str]      =   self.image_processor.size['''longest_edge''']




			def __call__(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__ = None,	**__magic_name__,	)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       Tuple      =   self.image_processor(
							    __UpperCamelCase,	return_tensors=__UpperCamelCase,	**__UpperCamelCase,	)

							# pop arguments that are not used in the foward but used nevertheless
							UpperCamelCase__							:       Optional[Any]      =   encoding_image_processor['''original_sizes''']

							if hasattr(__UpperCamelCase,	'''numpy'''							):  # Checks if Torch or TF tensor
											UpperCamelCase__							:       str      =   original_sizes.numpy()

							UpperCamelCase__     ,UpperCamelCase__     ,UpperCamelCase__							:       str      =   self._check_and_preprocess_points(
							    input_points=__UpperCamelCase,	input_labels=__UpperCamelCase,	input_boxes=__UpperCamelCase,	)

							UpperCamelCase__							:       Union[str, Any]      =   self._normalize_and_convert(
							    __UpperCamelCase,	__UpperCamelCase,	input_points=__UpperCamelCase,	input_labels=__UpperCamelCase,	input_boxes=__UpperCamelCase,	return_tensors=__UpperCamelCase,	)

							return encoding_image_processor




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__="pt",	)				->   Dict:






							"""simple docstring"""


							if input_points is not None:
											if len(__UpperCamelCase							) != len(__UpperCamelCase							):
															UpperCamelCase__							:       List[str]      =   [
															    self._normalize_coordinates(self.target_size,	__UpperCamelCase,	original_sizes[0]							) for point in input_points
															]
											else:
															UpperCamelCase__							:       Any      =   [
															    self._normalize_coordinates(self.target_size,	__UpperCamelCase,	__UpperCamelCase							)
															    for point, original_size in zip(__UpperCamelCase,	__UpperCamelCase							)
															]
											# check that all arrays have the same shape
											if not all(point.shape == input_points[0].shape for point in input_points							):
															if input_labels is not None:
																			UpperCamelCase__     ,UpperCamelCase__							:       int      =   self._pad_points_and_labels(__UpperCamelCase,	__UpperCamelCase							)

											UpperCamelCase__							:       Union[str, Any]      =   np.array(__UpperCamelCase							)

							if input_labels is not None:
											UpperCamelCase__							:       Dict      =   np.array(__UpperCamelCase							)

							if input_boxes is not None:
											if len(__UpperCamelCase							) != len(__UpperCamelCase							):
															UpperCamelCase__							:       Any      =   [
															    self._normalize_coordinates(self.target_size,	__UpperCamelCase,	original_sizes[0],	is_bounding_box=__UpperCamelCase							)
															    for box in input_boxes
															]
											else:
															UpperCamelCase__							:       str      =   [
															    self._normalize_coordinates(self.target_size,	__UpperCamelCase,	__UpperCamelCase,	is_bounding_box=__UpperCamelCase							)
															    for box, original_size in zip(__UpperCamelCase,	__UpperCamelCase							)
															]
											UpperCamelCase__							:       Tuple      =   np.array(__UpperCamelCase							)

							if input_boxes is not None:
											if return_tensors == "pt":
															UpperCamelCase__							:       Dict      =   torch.from_numpy(__UpperCamelCase							)
															# boxes batch size of 1 by default
															UpperCamelCase__							:       Dict      =   input_boxes.unsqueeze(1							) if len(input_boxes.shape							) != 3 else input_boxes
											elif return_tensors == "tf":
															UpperCamelCase__							:       Tuple      =   tf.convert_to_tensor(__UpperCamelCase							)
															# boxes batch size of 1 by default
															UpperCamelCase__							:       Union[str, Any]      =   tf.expand_dims(__UpperCamelCase,	1							) if len(input_boxes.shape							) != 3 else input_boxes
											encoding_image_processor.update({'''input_boxes''': input_boxes}							)
							if input_points is not None:
											if return_tensors == "pt":
															UpperCamelCase__							:       List[str]      =   torch.from_numpy(__UpperCamelCase							)
															# point batch size of 1 by default
															UpperCamelCase__							:       Optional[int]      =   input_points.unsqueeze(1							) if len(input_points.shape							) != 4 else input_points
											elif return_tensors == "tf":
															UpperCamelCase__							:       str      =   tf.convert_to_tensor(__UpperCamelCase							)
															# point batch size of 1 by default
															UpperCamelCase__							:       Dict      =   tf.expand_dims(__UpperCamelCase,	1							) if len(input_points.shape							) != 4 else input_points
											encoding_image_processor.update({'''input_points''': input_points}							)
							if input_labels is not None:
											if return_tensors == "pt":
															UpperCamelCase__							:       str      =   torch.from_numpy(__UpperCamelCase							)
															# point batch size of 1 by default
															UpperCamelCase__							:       Tuple      =   input_labels.unsqueeze(1							) if len(input_labels.shape							) != 3 else input_labels
											elif return_tensors == "tf":
															UpperCamelCase__							:       Union[str, Any]      =   tf.convert_to_tensor(__UpperCamelCase							)
															# point batch size of 1 by default
															UpperCamelCase__							:       List[str]      =   tf.expand_dims(__UpperCamelCase,	1							) if len(input_labels.shape							) != 3 else input_labels
											encoding_image_processor.update({'''input_labels''': input_labels}							)

							return encoding_image_processor




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__							)				->   List[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       str      =   max([point.shape[0] for point in input_points]							)
							UpperCamelCase__							:       int      =   []
							for i, point in enumerate(__UpperCamelCase							):
											if point.shape[0] != expected_nb_points:
															UpperCamelCase__							:       List[Any]      =   np.concatenate(
															    [point, np.zeros((expected_nb_points - point.shape[0], 2)							) + self.point_pad_value],	axis=0							)
															UpperCamelCase__							:       Any      =   np.append(input_labels[i],	[self.point_pad_value]							)
											processed_input_points.append(__UpperCamelCase							)
							UpperCamelCase__							:       List[str]      =   processed_input_points
							return input_points, input_labels




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__=False							)				->   Any:






							"""simple docstring"""
							UpperCamelCase__     ,UpperCamelCase__							:       Optional[int]      =   original_size
							UpperCamelCase__     ,UpperCamelCase__							:       Optional[Any]      =   self.image_processor._get_preprocess_shape(__UpperCamelCase,	longest_edge=__UpperCamelCase							)
							UpperCamelCase__							:       Union[str, Any]      =   deepcopy(__UpperCamelCase							).astype(__UpperCamelCase							)

							if is_bounding_box:
											UpperCamelCase__							:       Optional[Any]      =   coords.reshape(-1,	2,	2							)

							UpperCamelCase__							:       Union[str, Any]      =   coords[..., 0] * (new_w / old_w)
							UpperCamelCase__							:       Optional[Any]      =   coords[..., 1] * (new_h / old_h)

							if is_bounding_box:
											UpperCamelCase__							:       List[Any]      =   coords.reshape(-1,	4							)

							return coords




			def    UpperCamelCase__						(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	)				->   List[str]:






							"""simple docstring"""


							if input_points is not None:
											if hasattr(__UpperCamelCase,	'''numpy'''							):  # Checks for TF or Torch tensor
															UpperCamelCase__							:       int      =   input_points.numpy().tolist()

											if not isinstance(__UpperCamelCase,	__UpperCamelCase							) or not isinstance(input_points[0],	__UpperCamelCase							):
															raise ValueError('''Input points must be a list of list of floating points.'''							)
											UpperCamelCase__							:       List[Any]      =   [np.array(__UpperCamelCase							) for input_point in input_points]
							else:
											UpperCamelCase__							:       Any      =   None

							if input_labels is not None:
											if hasattr(__UpperCamelCase,	'''numpy'''							):
															UpperCamelCase__							:       List[Any]      =   input_labels.numpy().tolist()

											if not isinstance(__UpperCamelCase,	__UpperCamelCase							) or not isinstance(input_labels[0],	__UpperCamelCase							):
															raise ValueError('''Input labels must be a list of list integers.'''							)
											UpperCamelCase__							:       List[str]      =   [np.array(__UpperCamelCase							) for label in input_labels]
							else:
											UpperCamelCase__							:       int      =   None

							if input_boxes is not None:
											if hasattr(__UpperCamelCase,	'''numpy'''							):
															UpperCamelCase__							:       Any      =   input_boxes.numpy().tolist()

											if (
											    not isinstance(__UpperCamelCase,	__UpperCamelCase							)
											    or not isinstance(input_boxes[0],	__UpperCamelCase							)
											    or not isinstance(input_boxes[0][0],	__UpperCamelCase							)
											):
															raise ValueError('''Input boxes must be a list of list of list of floating points.'''							)
											UpperCamelCase__							:       Optional[int]      =   [np.array(__UpperCamelCase							).astype(np.floataa							) for box in input_boxes]
							else:
											UpperCamelCase__							:       List[str]      =   None

							return input_points, input_labels, input_boxes




			@property
			def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






							"""simple docstring"""
							UpperCamelCase__							:       int      =   self.image_processor.model_input_names
							return list(dict.fromkeys(__UpperCamelCase							)							)




			def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   List[Any]:






							"""simple docstring"""
							return self.image_processor.post_process_masks(*__UpperCamelCase,	**__UpperCamelCase							)








import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase_  							=				logging.get_logger(__name__)

UpperCAmelCase_  							=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_  							=				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_  							=				{
    'Salesforce/codegen-350M-mono': 2048,
}






class 					lowercase__ (			__lowerCamelCase       ):

   '''simple docstring'''


   a		:							Union[str, Any]     	=	VOCAB_FILES_NAMES
   a		:							List[str]     	=	PRETRAINED_VOCAB_FILES_MAP
   a		:							Optional[int]     	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a		:							List[Any]     	=	["input_ids", "attention_mask"]
   a		:							Any     	=	CodeGenTokenizer




   def __init__(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__=False,	**__magic_name__,	)				->   Optional[int]:






       """simple docstring"""
       super().__init__(
           __magic_name__,	__magic_name__,	tokenizer_file=__magic_name__,	unk_token=__magic_name__,	bos_token=__magic_name__,	eos_token=__magic_name__,	add_prefix_space=__magic_name__,	**__magic_name__,	)

       if kwargs.pop('''add_bos_token''',	__magic_name__							):
           UpperCamelCase__							:       List[Any]      =   kwargs.pop('''name_or_path''',	''''''							)
           raise ValueError(
               '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
               '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
               f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
               f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
               '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
               ''' so that the fast tokenizer works correctly.'''							)

       UpperCamelCase__							:       Dict      =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',	__magic_name__							) != add_prefix_space:
           UpperCamelCase__							:       List[str]      =   getattr(__magic_name__,	pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:       int      =   add_prefix_space
           UpperCamelCase__							:       List[str]      =   pre_tok_class(**__magic_name__							)

       UpperCamelCase__							:       List[Any]      =   add_prefix_space




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   kwargs.get('''is_split_into_words''',	__magic_name__							)
       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   kwargs.get('''is_split_into_words''',	__magic_name__							)

       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Tuple[str]:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   self._tokenizer.model.save(__magic_name__,	name=__magic_name__							)
       return tuple(__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = False,	__magic_name__ = None,	__magic_name__ = None,	**__magic_name__,	)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   super().decode(
           token_ids=__magic_name__,	skip_special_tokens=__magic_name__,	clean_up_tokenization_spaces=__magic_name__,	**__magic_name__,	)

       if truncate_before_pattern is not None and len(__magic_name__							) > 0:
           UpperCamelCase__							:       str      =   self.truncate(__magic_name__,	__magic_name__							)

       return decoded_text




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__							)				->   List[str]:






       """simple docstring"""
       def find_re(__magic_name__,	__magic_name__,	__magic_name__							):
           UpperCamelCase__							:       Dict      =   pattern.search(__magic_name__,	__magic_name__							)
           return m.start() if m else -1

       UpperCamelCase__							:       Union[str, Any]      =   [re.compile(__magic_name__,	re.MULTILINE							) for pattern in truncate_before_pattern]

       UpperCamelCase__							:       List[Any]      =   list(re.finditer('''^print''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Tuple      =   completion[: prints[1].start()]

       UpperCamelCase__							:       Optional[int]      =   list(re.finditer('''^def''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Optional[Any]      =   completion[: defs[1].start()]

       UpperCamelCase__							:       str      =   0

       UpperCamelCase__							:       Tuple      =   [
           pos for pos in [find_re(__magic_name__,	__magic_name__,	__magic_name__							) for terminal in terminals] if pos != -1
       ]

       if len(__magic_name__							) > 0:
           return completion[: min(__magic_name__							)]
       else:
           return completion

