


import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
							import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_						:			Optional[int]													=							logging.get_logger(__name__)

snake_case_						:			str													=							{'''vocab_file''': '''vocab.txt'''}

snake_case_						:			List[str]													=							{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

snake_case_						:			str													=							{
    '''openbmb/cpm-ant-10b''': 10_24,
}







def 						lowerCamelCase(   a__):
				_SCREAMING_SNAKE_CASE			  =collections.OrderedDict()
				with open(a__	,'''r'''	,encoding='''utf-8''') as reader:
								_SCREAMING_SNAKE_CASE			  =reader.readlines()
				for index, token in enumerate(a__):
								_SCREAMING_SNAKE_CASE			  =token.rstrip('''\n''')
								_SCREAMING_SNAKE_CASE			  =index
				return vocab

class 						A__							(		UpperCamelCase__  ):




						def __init__(       self					:      Optional[Any]				,    _a					:      Tuple				,    _a					:      Optional[Any]="<unk>"				,    _a					:      Optional[int]=200  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =vocab
										_SCREAMING_SNAKE_CASE			  =unk_token
										_SCREAMING_SNAKE_CASE			  =max_input_chars_per_word




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      Any  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =list(_a  )
										if len(_a  ) > self.max_input_chars_per_word:
														return [self.unk_token]

										_SCREAMING_SNAKE_CASE			  =0
										_SCREAMING_SNAKE_CASE			  =[]
										while start < len(_a  ):
														_SCREAMING_SNAKE_CASE			  =len(_a  )
														_SCREAMING_SNAKE_CASE			  =None
														while start < end:
																		_SCREAMING_SNAKE_CASE			  =''''''.join(chars[start:end]  )
																		if substr in self.vocab:
																						_SCREAMING_SNAKE_CASE			  =substr
																						break
																		end -= 1
														if cur_substr is None:
																		sub_tokens.append(self.unk_token  )
																		start += 1
														else:
																		sub_tokens.append(_a  )
																		_SCREAMING_SNAKE_CASE			  =end

										return sub_tokens

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						VOCAB_FILES_NAMES
						UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase             =						["input_ids", "attention_mask"]
						UpperCAmelCase             =						False




						def __init__(       self					:      str				,    _a					:      int				,    _a					:      Optional[int]="<d>"				,    _a					:      Any="</d>"				,    _a					:      Optional[int]="<s>"				,    _a					:      List[Any]="</s>"				,    _a					:      Tuple="<pad>"				,    _a					:      Tuple="<unk>"				,    _a					:      List[Any]="</n>"				,    _a					:      int="</_>"				,    _a					:      Optional[int]="left"				,    **_a					:      List[Any]				,    )							->     List[str]:


										"""simple docstring"""





										requires_backends(self				,    ['''jieba''']  )
										super().__init__(
										    bod_token=_a				,    eod_token=_a				,    bos_token=_a				,    eos_token=_a				,    pad_token=_a				,    unk_token=_a				,    line_token=_a				,    space_token=_a				,    padding_side=_a				,    **_a				,    )
										_SCREAMING_SNAKE_CASE			  =bod_token
										_SCREAMING_SNAKE_CASE			  =eod_token
										_SCREAMING_SNAKE_CASE			  =load_vocab(_a  )
										_SCREAMING_SNAKE_CASE			  =self.encoder[space_token]
										_SCREAMING_SNAKE_CASE			  =self.encoder[line_token]

										del self.encoder[space_token]
										del self.encoder[line_token]

										_SCREAMING_SNAKE_CASE			  =collections.OrderedDict(sorted(self.encoder.items()				,    key=lambda _a  : x[1]  )  )
										_SCREAMING_SNAKE_CASE			  ={v: k for k, v in self.encoder.items()}

										_SCREAMING_SNAKE_CASE			  =WordpieceTokenizer(vocab=self.encoder				,    unk_token=self.unk_token  )




						@property
						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     str:


										"""simple docstring"""





										return self.encoder[self.bod_token]




						@property
						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     Optional[int]:


										"""simple docstring"""





										return self.encoder[self.eod_token]




						@property
						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										return self.encoder["\n"]




						@property
						def 							__UpperCamelCase				(       self					:      List[str]  )							->     int:


										"""simple docstring"""





										return len(self.encoder  )




						def 							__UpperCamelCase				(       self					:      int  )							->     Optional[Any]:


										"""simple docstring"""





										return dict(self.encoder				,    **self.added_tokens_encoder  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Tuple  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[]
										for x in jieba.cut(_a				,    cut_all=_a  ):
														output_tokens.extend(self.wordpiece_tokenizer.tokenize(_a  )  )
										return output_tokens




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      Dict				,    **_a					:      Dict  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[i for i in token_ids if i >= 0]
										_SCREAMING_SNAKE_CASE			  =[
										    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
										]
										return super()._decode(_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      Optional[Any]  )							->     Optional[Any]:


										"""simple docstring"""





										return token in self.encoder




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      List[str]  )							->     str:


										"""simple docstring"""





										return "".join(_a  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      Tuple  )							->     Union[str, Any]:


										"""simple docstring"""





										return self.encoder.get(_a				,    self.encoder.get(self.unk_token  )  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										return self.decoder.get(_a				,    self.unk_token  )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


										"""simple docstring"""





										if os.path.isdir(_a  ):
														_SCREAMING_SNAKE_CASE			  =os.path.join(
														    _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
										else:
														_SCREAMING_SNAKE_CASE			  =(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
										_SCREAMING_SNAKE_CASE			  =0
										if " " in self.encoder:
														_SCREAMING_SNAKE_CASE			  =self.encoder[''' ''']
														del self.encoder[" "]
										if "\n" in self.encoder:
														_SCREAMING_SNAKE_CASE			  =self.encoder['''\n''']
														del self.encoder["\n"]
										_SCREAMING_SNAKE_CASE			  =collections.OrderedDict(sorted(self.encoder.items()				,    key=lambda _a  : x[1]  )  )
										with open(_a				,    '''w'''				,    encoding='''utf-8'''  ) as writer:
														for token, token_index in self.encoder.items():
																		if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																						    ''' Please check that the vocabulary is not corrupted!'''  )
																						_SCREAMING_SNAKE_CASE			  =token_index
																		writer.write(token + '''\n'''  )
																		index += 1
										return (vocab_file,)




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      List[int]				,    _a					:      List[int] = None  )							->     List[int]:


										"""simple docstring"""





										if token_ids_a is None:
														return [self.bos_token_id] + token_ids_a
										return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None				,    _a					:      bool = False  )							->     List[int]:


										"""simple docstring"""





										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_a				,    token_ids_a=_a				,    already_has_special_tokens=_a  )

										if token_ids_a is not None:
														return [1] + ([0] * len(_a  )) + [1] + ([0] * len(_a  ))
										return [1] + ([0] * len(_a  ))



from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

snake_case_						:			Union[str, Any]													=							logging.get_logger(__name__)







def 						lowerCamelCase(   a__):
				if isinstance(a__	,(list, tuple)) and isinstance(videos[0]	,(list, tuple)) and is_valid_image(videos[0][0]):
								return videos

				elif isinstance(a__	,(list, tuple)) and is_valid_image(videos[0]):
								return [videos]

				elif is_valid_image(a__):
								return [[videos]]

				raise ValueError(f"Could not make batched video from {videos}")


class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						["pixel_values"]




						def __init__(       self					:      Tuple				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      bool = True				,    _a					:      Union[int, float] = 1 / 255				,    _a					:      bool = True				,    _a					:      bool = True				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    **_a					:      Any				,    )							->     None:


										"""simple docstring"""





										super().__init__(**_a  )
										_SCREAMING_SNAKE_CASE			  =size if size is not None else {'''shortest_edge''': 256}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										_SCREAMING_SNAKE_CASE			  =do_resize
										_SCREAMING_SNAKE_CASE			  =size
										_SCREAMING_SNAKE_CASE			  =do_center_crop
										_SCREAMING_SNAKE_CASE			  =crop_size
										_SCREAMING_SNAKE_CASE			  =resample
										_SCREAMING_SNAKE_CASE			  =do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset
										_SCREAMING_SNAKE_CASE			  =do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else IMAGENET_STANDARD_STD




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[Any]				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										if "shortest_edge" in size:
														_SCREAMING_SNAKE_CASE			  =get_resize_output_image_size(_a				,    size['''shortest_edge''']				,    default_to_square=_a  )
										elif "height" in size and "width" in size:
														_SCREAMING_SNAKE_CASE			  =(size['''height'''], size['''width'''])
										else:
														raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"  )
										return resize(_a				,    size=_a				,    resample=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Dict				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a  )
										if "height" not in size or "width" not in size:
														raise ValueError(f"Size must have 'height' and 'width' as keys. Got {size.keys()}"  )
										return center_crop(_a				,    size=(size['''height'''], size['''width'''])				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      np.ndarray				,    _a					:      Union[int, float]				,    _a					:      bool = True				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[str]				,    )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =image.astype(np.floataa  )
										if offset:
														_SCREAMING_SNAKE_CASE			  =image - (scale / 2)
										return rescale(_a				,    scale=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      np.ndarray				,    _a					:      Union[float, List[float]]				,    _a					:      Union[float, List[float]]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Any				,    )							->     np.ndarray:


										"""simple docstring"""





										return normalize(_a				,    mean=_a				,    std=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[ChannelDimension] = ChannelDimension.FIRST				,    )							->     np.ndarray:


										"""simple docstring"""





										if do_resize and size is None or resample is None:
														raise ValueError('''Size and resample must be specified if do_resize is True.'''  )

										if do_center_crop and crop_size is None:
														raise ValueError('''Crop size must be specified if do_center_crop is True.'''  )

										if do_rescale and rescale_factor is None:
														raise ValueError('''Rescale factor must be specified if do_rescale is True.'''  )

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError('''Image mean and std must be specified if do_normalize is True.'''  )

										if offset and not do_rescale:
														raise ValueError('''For offset, do_rescale must also be set to True.'''  )

										# All transformations expect numpy arrays.
										_SCREAMING_SNAKE_CASE			  =to_numpy_array(_a  )

										if do_resize:
														_SCREAMING_SNAKE_CASE			  =self.resize(image=_a				,    size=_a				,    resample=_a  )

										if do_center_crop:
														_SCREAMING_SNAKE_CASE			  =self.center_crop(_a				,    size=_a  )

										if do_rescale:
														_SCREAMING_SNAKE_CASE			  =self.rescale(image=_a				,    scale=_a				,    offset=_a  )

										if do_normalize:
														_SCREAMING_SNAKE_CASE			  =self.normalize(image=_a				,    mean=_a				,    std=_a  )

										_SCREAMING_SNAKE_CASE			  =to_channel_dimension_format(_a				,    _a  )
										return image




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[str, TensorType]] = None				,    _a					:      ChannelDimension = ChannelDimension.FIRST				,    **_a					:      str				,    )							->     PIL.Image.Image:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =do_resize if do_resize is not None else self.do_resize
										_SCREAMING_SNAKE_CASE			  =resample if resample is not None else self.resample
										_SCREAMING_SNAKE_CASE			  =do_center_crop if do_center_crop is not None else self.do_center_crop
										_SCREAMING_SNAKE_CASE			  =do_rescale if do_rescale is not None else self.do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor if rescale_factor is not None else self.rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset if offset is not None else self.offset
										_SCREAMING_SNAKE_CASE			  =do_normalize if do_normalize is not None else self.do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else self.image_mean
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else self.image_std

										_SCREAMING_SNAKE_CASE			  =size if size is not None else self.size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else self.crop_size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										if not valid_images(_a  ):
														raise ValueError(
														    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
														    '''torch.Tensor, tf.Tensor or jax.ndarray.'''  )

										_SCREAMING_SNAKE_CASE			  =make_batched(_a  )

										_SCREAMING_SNAKE_CASE			  =[
										    [
										        self._preprocess_image(
										            image=_a				,    do_resize=_a				,    size=_a				,    resample=_a				,    do_center_crop=_a				,    crop_size=_a				,    do_rescale=_a				,    rescale_factor=_a				,    offset=_a				,    do_normalize=_a				,    image_mean=_a				,    image_std=_a				,    data_format=_a				,    )
										        for img in video
										    ]
										    for video in videos
										]

										_SCREAMING_SNAKE_CASE			  ={'''pixel_values''': videos}
										return BatchFeature(data=_a				,    tensor_type=_a  )