
from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
		import torch

if is_tf_available():
		import tensorflow as tf





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    ['''image_processor''']
					SCREAMING_SNAKE_CASE			    =    '''SamImageProcessor'''

					def __init__(	self					:     Any   ,					__UpperCamelCase					:     Tuple   ):
												super().__init__(__UpperCamelCase   )
												lowerCamelCase_							  =					self.image_processor
												lowerCamelCase_							  =					-1_0
												lowerCamelCase_							  =					self.image_processor.size["""longest_edge"""]

					def __call__(	self					:     Optional[Any]   ,					__UpperCamelCase					:     Optional[Any]=None   ,					__UpperCamelCase					:     Dict=None   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     Optional[Union[str, TensorType]] = None   ,					**__UpperCamelCase					:     Union[str, Any]   ,					):
												lowerCamelCase_							  =					self.image_processor(
												    __UpperCamelCase   ,					return_tensors=__UpperCamelCase   ,					**__UpperCamelCase   ,					)

												# pop arguments that are not used in the foward but used nevertheless
												lowerCamelCase_							  =					encoding_image_processor["""original_sizes"""]

												if hasattr(__UpperCamelCase   ,					"""numpy"""   ):  # Checks if Torch or TF tensor
																			lowerCamelCase_							  =					original_sizes.numpy()

												lowerCamelCase_  ,   lowerCamelCase_  ,   lowerCamelCase_							  =					self._check_and_preprocess_points(
												    input_points=__UpperCamelCase   ,					input_labels=__UpperCamelCase   ,					input_boxes=__UpperCamelCase   ,					)

												lowerCamelCase_							  =					self._normalize_and_convert(
												    __UpperCamelCase   ,					__UpperCamelCase   ,					input_points=__UpperCamelCase   ,					input_labels=__UpperCamelCase   ,					input_boxes=__UpperCamelCase   ,					return_tensors=__UpperCamelCase   ,					)

												return encoding_image_processor

					def    lowercase__      (	self					:     List[Any]   ,					__UpperCamelCase					:     List[str]   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     List[str]=None   ,					__UpperCamelCase					:     int=None   ,					__UpperCamelCase					:     Any=None   ,					__UpperCamelCase					:     str="pt"   ,					):
												if input_points is not None:
																			if len(__UpperCamelCase   ) != len(__UpperCamelCase   ):
																										lowerCamelCase_							  =					[
																										    self._normalize_coordinates(self.target_size   ,					__UpperCamelCase   ,					original_sizes[0]   ) for point in input_points
																										]
																			else:
																										lowerCamelCase_							  =					[
																										    self._normalize_coordinates(self.target_size   ,					__UpperCamelCase   ,					__UpperCamelCase   )
																										    for point, original_size in zip(__UpperCamelCase   ,					__UpperCamelCase   )
																										]
																			# check that all arrays have the same shape
																			if not all(point.shape == input_points[0].shape for point in input_points   ):
																										if input_labels is not None:
																																	lowerCamelCase_  ,   lowerCamelCase_							  =					self._pad_points_and_labels(__UpperCamelCase   ,					__UpperCamelCase   )

																			lowerCamelCase_							  =					np.array(__UpperCamelCase   )

												if input_labels is not None:
																			lowerCamelCase_							  =					np.array(__UpperCamelCase   )

												if input_boxes is not None:
																			if len(__UpperCamelCase   ) != len(__UpperCamelCase   ):
																										lowerCamelCase_							  =					[
																										    self._normalize_coordinates(self.target_size   ,					__UpperCamelCase   ,					original_sizes[0]   ,					is_bounding_box=__UpperCamelCase   )
																										    for box in input_boxes
																										]
																			else:
																										lowerCamelCase_							  =					[
																										    self._normalize_coordinates(self.target_size   ,					__UpperCamelCase   ,					__UpperCamelCase   ,					is_bounding_box=__UpperCamelCase   )
																										    for box, original_size in zip(__UpperCamelCase   ,					__UpperCamelCase   )
																										]
																			lowerCamelCase_							  =					np.array(__UpperCamelCase   )

												if input_boxes is not None:
																			if return_tensors == "pt":
																										lowerCamelCase_							  =					torch.from_numpy(__UpperCamelCase   )
																										# boxes batch size of 1 by default
																										lowerCamelCase_							  =					input_boxes.unsqueeze(1   ) if len(input_boxes.shape   ) != 3 else input_boxes
																			elif return_tensors == "tf":
																										lowerCamelCase_							  =					tf.convert_to_tensor(__UpperCamelCase   )
																										# boxes batch size of 1 by default
																										lowerCamelCase_							  =					tf.expand_dims(__UpperCamelCase   ,					1   ) if len(input_boxes.shape   ) != 3 else input_boxes
																			encoding_image_processor.update({"""input_boxes""": input_boxes}   )
												if input_points is not None:
																			if return_tensors == "pt":
																										lowerCamelCase_							  =					torch.from_numpy(__UpperCamelCase   )
																										# point batch size of 1 by default
																										lowerCamelCase_							  =					input_points.unsqueeze(1   ) if len(input_points.shape   ) != 4 else input_points
																			elif return_tensors == "tf":
																										lowerCamelCase_							  =					tf.convert_to_tensor(__UpperCamelCase   )
																										# point batch size of 1 by default
																										lowerCamelCase_							  =					tf.expand_dims(__UpperCamelCase   ,					1   ) if len(input_points.shape   ) != 4 else input_points
																			encoding_image_processor.update({"""input_points""": input_points}   )
												if input_labels is not None:
																			if return_tensors == "pt":
																										lowerCamelCase_							  =					torch.from_numpy(__UpperCamelCase   )
																										# point batch size of 1 by default
																										lowerCamelCase_							  =					input_labels.unsqueeze(1   ) if len(input_labels.shape   ) != 3 else input_labels
																			elif return_tensors == "tf":
																										lowerCamelCase_							  =					tf.convert_to_tensor(__UpperCamelCase   )
																										# point batch size of 1 by default
																										lowerCamelCase_							  =					tf.expand_dims(__UpperCamelCase   ,					1   ) if len(input_labels.shape   ) != 3 else input_labels
																			encoding_image_processor.update({"""input_labels""": input_labels}   )

												return encoding_image_processor

					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     Any   ,					__UpperCamelCase					:     Any   ):
												lowerCamelCase_							  =					max([point.shape[0] for point in input_points]   )
												lowerCamelCase_							  =					[]
												for i, point in enumerate(__UpperCamelCase   ):
																			if point.shape[0] != expected_nb_points:
																										lowerCamelCase_							  =					np.concatenate(
																										    [point, np.zeros((expected_nb_points - point.shape[0], 2)   ) + self.point_pad_value]   ,					axis=0   )
																										lowerCamelCase_							  =					np.append(input_labels[i]   ,					[self.point_pad_value]   )
																			processed_input_points.append(__UpperCamelCase   )
												lowerCamelCase_							  =					processed_input_points
												return input_points, input_labels

					def    lowercase__      (	self					:     Optional[int]   ,					__UpperCamelCase					:     int   ,					__UpperCamelCase					:     np.ndarray   ,					__UpperCamelCase					:     Optional[Any]   ,					__UpperCamelCase					:     Dict=False   ):
												lowerCamelCase_  ,   lowerCamelCase_							  =					original_size
												lowerCamelCase_  ,   lowerCamelCase_							  =					self.image_processor._get_preprocess_shape(__UpperCamelCase   ,					longest_edge=__UpperCamelCase   )
												lowerCamelCase_							  =					deepcopy(__UpperCamelCase   ).astype(__UpperCamelCase   )

												if is_bounding_box:
																			lowerCamelCase_							  =					coords.reshape(-1   ,					2   ,					2   )

												lowerCamelCase_							  =					coords[..., 0] * (new_w / old_w)
												lowerCamelCase_							  =					coords[..., 1] * (new_h / old_h)

												if is_bounding_box:
																			lowerCamelCase_							  =					coords.reshape(-1   ,					4   )

												return coords

					def    lowercase__      (	self					:     List[Any]   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     Any=None   ,					__UpperCamelCase					:     Union[str, Any]=None   ,					):
												if input_points is not None:
																			if hasattr(__UpperCamelCase   ,					"""numpy"""   ):  # Checks for TF or Torch tensor
																										lowerCamelCase_							  =					input_points.numpy().tolist()

																			if not isinstance(__UpperCamelCase   ,					__UpperCamelCase   ) or not isinstance(input_points[0]   ,					__UpperCamelCase   ):
																										raise ValueError("""Input points must be a list of list of floating points."""   )
																			lowerCamelCase_							  =					[np.array(__UpperCamelCase   ) for input_point in input_points]
												else:
																			lowerCamelCase_							  =					None

												if input_labels is not None:
																			if hasattr(__UpperCamelCase   ,					"""numpy"""   ):
																										lowerCamelCase_							  =					input_labels.numpy().tolist()

																			if not isinstance(__UpperCamelCase   ,					__UpperCamelCase   ) or not isinstance(input_labels[0]   ,					__UpperCamelCase   ):
																										raise ValueError("""Input labels must be a list of list integers."""   )
																			lowerCamelCase_							  =					[np.array(__UpperCamelCase   ) for label in input_labels]
												else:
																			lowerCamelCase_							  =					None

												if input_boxes is not None:
																			if hasattr(__UpperCamelCase   ,					"""numpy"""   ):
																										lowerCamelCase_							  =					input_boxes.numpy().tolist()

																			if (
																			    not isinstance(__UpperCamelCase   ,					__UpperCamelCase   )
																			    or not isinstance(input_boxes[0]   ,					__UpperCamelCase   )
																			    or not isinstance(input_boxes[0][0]   ,					__UpperCamelCase   )
																			):
																										raise ValueError("""Input boxes must be a list of list of list of floating points."""   )
																			lowerCamelCase_							  =					[np.array(__UpperCamelCase   ).astype(np.floataa   ) for box in input_boxes]
												else:
																			lowerCamelCase_							  =					None

												return input_points, input_labels, input_boxes

					@property
					def    lowercase__      (	self					:     List[str]   ):
												lowerCamelCase_							  =					self.image_processor.model_input_names
												return list(dict.fromkeys(__UpperCamelCase   )   )



					def    lowercase__      (	self					:     Tuple   ,					*__UpperCamelCase					:     List[str]   ,					**__UpperCamelCase					:     Optional[int]   ):
												return self.image_processor.post_process_masks(*__UpperCamelCase   ,					**__UpperCamelCase   )







import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.txt'''}

lowercase        =  {
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

lowercase        =  {
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[str]       )  ->    Dict:
							lowerCamelCase_							  =					collections.OrderedDict()
							with open(UpperCAmelCase__			,				"""r"""			,				encoding="""utf-8"""       ) as reader:
														lowerCamelCase_							  =					reader.readlines()
							for index, token in enumerate(UpperCAmelCase__       ):
														lowerCamelCase_							  =					token.rstrip("""\n"""       )
														lowerCamelCase_							  =					index
							return vocab





class       __A(				UpperCAmelCase       ):

					def __init__(	self					:     Optional[Any]   ,					__UpperCamelCase					:     Tuple   ,					__UpperCamelCase					:     List[str]="<unk>"   ,					__UpperCamelCase					:     List[str]=2_0_0   ):
												lowerCamelCase_							  =					vocab
												lowerCamelCase_							  =					unk_token
												lowerCamelCase_							  =					max_input_chars_per_word



					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     int   ):
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												if len(__UpperCamelCase   ) > self.max_input_chars_per_word:
																			return [self.unk_token]

												lowerCamelCase_							  =					0
												lowerCamelCase_							  =					[]
												while start < len(__UpperCamelCase   ):
																			lowerCamelCase_							  =					len(__UpperCamelCase   )
																			lowerCamelCase_							  =					None
																			while start < end:
																										lowerCamelCase_							  =					"""""".join(chars[start:end]   )
																										if substr in self.vocab:
																																	lowerCamelCase_							  =					substr
																																	break
																										end -= 1
																			if cur_substr is None:
																										sub_tokens.append(self.unk_token   )
																										start += 1
																			else:
																										sub_tokens.append(__UpperCamelCase   )
																										lowerCamelCase_							  =					end

												return sub_tokens





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    ['''input_ids''', '''attention_mask''']
					SCREAMING_SNAKE_CASE			    =    False

					def __init__(	self					:     int   ,					__UpperCamelCase					:     Optional[int]   ,					__UpperCamelCase					:     Optional[int]="<d>"   ,					__UpperCamelCase					:     List[str]="</d>"   ,					__UpperCamelCase					:     Optional[Any]="<s>"   ,					__UpperCamelCase					:     Union[str, Any]="</s>"   ,					__UpperCamelCase					:     List[str]="<pad>"   ,					__UpperCamelCase					:     Union[str, Any]="<unk>"   ,					__UpperCamelCase					:     List[Any]="</n>"   ,					__UpperCamelCase					:     Tuple="</_>"   ,					__UpperCamelCase					:     Optional[Any]="left"   ,					**__UpperCamelCase					:     List[str]   ,					):
												requires_backends(self   ,					["""jieba"""]   )
												super().__init__(
												    bod_token=__UpperCamelCase   ,					eod_token=__UpperCamelCase   ,					bos_token=__UpperCamelCase   ,					eos_token=__UpperCamelCase   ,					pad_token=__UpperCamelCase   ,					unk_token=__UpperCamelCase   ,					line_token=__UpperCamelCase   ,					space_token=__UpperCamelCase   ,					padding_side=__UpperCamelCase   ,					**__UpperCamelCase   ,					)
												lowerCamelCase_							  =					bod_token
												lowerCamelCase_							  =					eod_token
												lowerCamelCase_							  =					load_vocab(__UpperCamelCase   )
												lowerCamelCase_							  =					self.encoder[space_token]
												lowerCamelCase_							  =					self.encoder[line_token]

												del self.encoder[space_token]
												del self.encoder[line_token]

												lowerCamelCase_							  =					collections.OrderedDict(sorted(self.encoder.items()   ,					key=lambda __UpperCamelCase   : x[1]   )   )
												lowerCamelCase_							  =					{v: k for k, v in self.encoder.items()}

												lowerCamelCase_							  =					WordpieceTokenizer(vocab=self.encoder   ,					unk_token=self.unk_token   )

					@property
					def    lowercase__      (	self					:     Any   ):
												return self.encoder[self.bod_token]

					@property
					def    lowercase__      (	self					:     Any   ):
												return self.encoder[self.eod_token]

					@property
					def    lowercase__      (	self					:     int   ):
												return self.encoder["\n"]

					@property
					def    lowercase__      (	self					:     str   ):
												return len(self.encoder   )

					def    lowercase__      (	self					:     Any   ):
												return dict(self.encoder   ,					**self.added_tokens_encoder   )

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     Optional[Any]   ):
												lowerCamelCase_							  =					[]
												for x in jieba.cut(__UpperCamelCase   ,					cut_all=__UpperCamelCase   ):
																			output_tokens.extend(self.wordpiece_tokenizer.tokenize(__UpperCamelCase   )   )
												return output_tokens

					def    lowercase__      (	self					:     Optional[Any]   ,					__UpperCamelCase					:     Dict   ,					**__UpperCamelCase					:     str   ):
												lowerCamelCase_							  =					[i for i in token_ids if i >= 0]
												lowerCamelCase_							  =					[
												    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
												]
												return super()._decode(__UpperCamelCase   ,					**__UpperCamelCase   )

					def    lowercase__      (	self					:     Dict   ,					__UpperCamelCase					:     Optional[Any]   ):
												return token in self.encoder

					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     List[str]   ):
												return "".join(__UpperCamelCase   )

					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     int   ):
												return self.encoder.get(__UpperCamelCase   ,					self.encoder.get(self.unk_token   )   )

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     Union[str, Any]   ):
												return self.decoder.get(__UpperCamelCase   ,					self.unk_token   )

					def    lowercase__      (	self					:     Union[str, Any]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[str] = None   ):
												if os.path.isdir(__UpperCamelCase   ):
																			lowerCamelCase_							  =					os.path.join(
																			    __UpperCamelCase   ,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
												else:
																			lowerCamelCase_							  =					(filename_prefix + """-""" if filename_prefix else """""") + save_directory
												lowerCamelCase_							  =					0
												if " " in self.encoder:
																			lowerCamelCase_							  =					self.encoder[""" """]
																			del self.encoder[" "]
												if "\n" in self.encoder:
																			lowerCamelCase_							  =					self.encoder["""\n"""]
																			del self.encoder["\n"]
												lowerCamelCase_							  =					collections.OrderedDict(sorted(self.encoder.items()   ,					key=lambda __UpperCamelCase   : x[1]   )   )
												with open(__UpperCamelCase   ,					"""w"""   ,					encoding="""utf-8"""   ) as writer:
																			for token, token_index in self.encoder.items():
																										if index != token_index:
																																	logger.warning(
																																	    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																																	    """ Please check that the vocabulary is not corrupted!"""   )
																																	lowerCamelCase_							  =					token_index
																										writer.write(token + """\n"""   )
																										index += 1
												return (vocab_file,)

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     List[int] = None   ):
												if token_ids_a is None:
																			return [self.bos_token_id] + token_ids_a
												return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     Optional[List[int]] = None   ,					__UpperCamelCase					:     bool = False   ):

												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__UpperCamelCase   ,					token_ids_a=__UpperCamelCase   ,					already_has_special_tokens=__UpperCamelCase   )

												if token_ids_a is not None:
																			return [1] + ([0] * len(__UpperCamelCase   )) + [1] + ([0] * len(__UpperCamelCase   ))
												return [1] + ([0] * len(__UpperCamelCase   ))





