import os
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken
from ...utils import logging


_lowercase:	Any       			=		logging.get_logger(__name__)

_lowercase:	str       			=		{'''vocab_file''': '''vocab.txt'''}

_lowercase:	Optional[int]       			=		{
    '''vocab_file''': {
        '''facebook/esm2_t6_8M_UR50D''': '''https://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txt''',
        '''facebook/esm2_t12_35M_UR50D''': '''https://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt''',
    },
}

_lowercase:	str       			=		{
    '''facebook/esm2_t6_8M_UR50D''': 1_0_2_4,
    '''facebook/esm2_t12_35M_UR50D''': 1_0_2_4,
}






def _lowerCamelCase  (		snake_case  ):
							with open(snake_case     , 'r'  ) as f:
														_lowerCAmelCase							=							f.read().splitlines()
														return [l.strip() for l in lines]



class        lowerCamelCase__				(   UpperCAmelCase				):
			UpperCamelCase__    							=VOCAB_FILES_NAMES
			UpperCamelCase__    							=PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__    							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__    							=["input_ids", "attention_mask"]





			def __init__(       self	:	List[Any]  ,  lowercase__	:	int  ,  lowercase__	:	Optional[Any]="<unk>"  ,  lowercase__	:	List[Any]="<cls>"  ,  lowercase__	:	List[Any]="<pad>"  ,  lowercase__	:	Tuple="<mask>"  ,  lowercase__	:	Union[str, Any]="<eos>"  ,  **lowercase__	:	int  ,  ):
										super().__init__(**lowercase__						)
										_lowerCAmelCase							=							load_vocab_file(lowercase__						)
										_lowerCAmelCase							=							dict(enumerate(self.all_tokens						)						)
										_lowerCAmelCase							=							{tok: ind for ind, tok in enumerate(self.all_tokens						)}
										_lowerCAmelCase							=							unk_token
										_lowerCAmelCase							=							cls_token
										_lowerCAmelCase							=							pad_token
										_lowerCAmelCase							=							mask_token
										_lowerCAmelCase							=							eos_token
										_lowerCAmelCase							=							self.all_tokens
										self._create_trie(self.unique_no_split_tokens						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[str]  ,  lowercase__	:	int						):
										return self._id_to_token.get(lowercase__  ,  self.unk_token						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]  ,  lowercase__	:	str						):
										return self._token_to_id.get(lowercase__  ,  self._token_to_id.get(self.unk_token						)						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]  ,  lowercase__	:	Dict  ,  **lowercase__	:	Optional[int]						):
										return text.split()





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[str]  ,  lowercase__	:	Optional[Any]=False						):
										return len(self._id_to_token						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										return {token: i for i, token in enumerate(self.all_tokens						)}





			def   SCREAMING_SNAKE_CASE__     (       self	:	int  ,  lowercase__	:	str						):
										return self._token_to_id.get(lowercase__  ,  self._token_to_id.get(self.unk_token						)						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple  ,  lowercase__	:	int						):
										return self._id_to_token.get(lowercase__  ,  self.unk_token						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[str]  ,  lowercase__	:	List[int]  ,  lowercase__	:	Optional[List[int]] = None						):
										_lowerCAmelCase							=							[self.cls_token_id]
										_lowerCAmelCase							=							[self.eos_token_id]  # No sep token in ESM vocabulary
										if token_ids_a is None:
																	if self.eos_token_id is None:
																								return cls + token_ids_a
																	else:
																								return cls + token_ids_a + sep
										elif self.eos_token_id is None:
																	raise ValueError('Cannot tokenize multiple sequences when EOS token is not set!'						)
										return cls + token_ids_a + sep + token_ids_a + sep  # Multiple inputs always have an EOS token





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[str]  ,  lowercase__	:	List  ,  lowercase__	:	Optional[List] = None  ,  lowercase__	:	bool = False						):
										if already_has_special_tokens:
																	if token_ids_a is not None:
																								raise ValueError(
																								    'You should not supply a second sequence if the provided sequence of '
																								    'ids is already formatted with special tokens for the model.'						)

																	return [1 if token in self.all_special_ids else 0 for token in token_ids_a]
										_lowerCAmelCase							=							[1] + ([0] * len(lowercase__						)) + [1]
										if token_ids_a is not None:
																	mask += [0] * len(lowercase__						) + [1]
										return mask





			def   SCREAMING_SNAKE_CASE__     (       self	:	Any  ,  lowercase__	:	Union[str, Any]  ,  lowercase__	:	Union[str, Any]						):
										_lowerCAmelCase							=							os.path.join(lowercase__  ,  (filename_prefix + '-' if filename_prefix else '') + 'vocab.txt'						)
										with open(lowercase__  ,  'w'						) as f:
																	f.write('\n'.join(self.all_tokens						)						)
										return (vocab_file,)





			@property
			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple						):
										return self.get_vocab_size(with_added_tokens=lowercase__						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	str  ,  lowercase__	:	Union[List[str], List[AddedToken]]  ,  lowercase__	:	bool = False						):
										return super()._add_tokens(lowercase__  ,  special_tokens=lowercase__						)







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import ChineseCLIPImageProcessor



class        lowerCamelCase__				(   unittest.TestCase				):





			def __init__(       self	:	List[str]  ,  lowercase__	:	Any  ,  lowercase__	:	List[Any]=7  ,  lowercase__	:	List[str]=3  ,  lowercase__	:	str=18  ,  lowercase__	:	List[Any]=30  ,  lowercase__	:	Optional[int]=4_00  ,  lowercase__	:	Dict=True  ,  lowercase__	:	List[str]=None  ,  lowercase__	:	int=True  ,  lowercase__	:	Tuple=None  ,  lowercase__	:	int=True  ,  lowercase__	:	Tuple=[0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3]  ,  lowercase__	:	Optional[int]=[0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1]  ,  lowercase__	:	Any=True  ,  ):
										_lowerCAmelCase							=							size if size is not None else {'height': 2_24, 'width': 2_24}
										_lowerCAmelCase							=							crop_size if crop_size is not None else {'height': 18, 'width': 18}
										_lowerCAmelCase							=							parent
										_lowerCAmelCase							=							batch_size
										_lowerCAmelCase							=							num_channels
										_lowerCAmelCase							=							image_size
										_lowerCAmelCase							=							min_resolution
										_lowerCAmelCase							=							max_resolution
										_lowerCAmelCase							=							do_resize
										_lowerCAmelCase							=							size
										_lowerCAmelCase							=							do_center_crop
										_lowerCAmelCase							=							crop_size
										_lowerCAmelCase							=							do_normalize
										_lowerCAmelCase							=							image_mean
										_lowerCAmelCase							=							image_std
										_lowerCAmelCase							=							do_convert_rgb





			def   SCREAMING_SNAKE_CASE__     (       self	:	Optional[Any]						):
										return {
										    "do_resize": self.do_resize,
										    "size": self.size,
										    "do_center_crop": self.do_center_crop,
										    "crop_size": self.crop_size,
										    "do_normalize": self.do_normalize,
										    "image_mean": self.image_mean,
										    "image_std": self.image_std,
										    "do_convert_rgb": self.do_convert_rgb,
										}





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[str]  ,  lowercase__	:	Tuple=False  ,  lowercase__	:	List[Any]=False  ,  lowercase__	:	str=False						):
										assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

										if equal_resolution:
																	_lowerCAmelCase							=							[]
																	for i in range(self.batch_size						):
																								image_inputs.append(
																								    np.random.randint(
																								        2_55  ,  size=(self.num_channels, self.max_resolution, self.max_resolution)  ,  dtype=np.uinta						)						)
										else:
																	_lowerCAmelCase							=							[]
																	for i in range(self.batch_size						):
																								_lowerCAmelCase			,		_lowerCAmelCase							=							np.random.choice(np.arange(self.min_resolution  ,  self.max_resolution						)  ,  2						)
																								image_inputs.append(np.random.randint(2_55  ,  size=(self.num_channels, width, height)  ,  dtype=np.uinta						)						)

										if not numpify and not torchify:
																	# PIL expects the channel dimension as last dimension
																	_lowerCAmelCase							=							[Image.fromarray(np.moveaxis(lowercase__  ,  0  ,  -1						)						) for x in image_inputs]

										if torchify:
																	_lowerCAmelCase							=							[torch.from_numpy(lowercase__						) for x in image_inputs]

										return image_inputs







@require_torch
@require_vision
class        lowerCamelCase__				(   UpperCAmelCase       ,unittest.TestCase				):
			UpperCamelCase__    							=ChineseCLIPImageProcessor if is_vision_available() else None





			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple						):
										_lowerCAmelCase							=							ChineseCLIPImageProcessingTester(self  ,  do_center_crop=lowercase__						)





			@property
			def   SCREAMING_SNAKE_CASE__     (       self	:	Any						):
										return self.image_processor_tester.prepare_image_processor_dict()





			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple						):
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										self.assertTrue(hasattr(lowercase__  ,  'do_resize'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'size'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_center_crop'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'center_crop'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_normalize'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'image_mean'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'image_std'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_convert_rgb'						)						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Union[str, Any]						):
										_lowerCAmelCase							=							self.image_processing_class.from_dict(self.image_processor_dict						)
										self.assertEqual(image_processor.size  ,  {'height': 2_24, 'width': 2_24}						)
										self.assertEqual(image_processor.crop_size  ,  {'height': 18, 'width': 18}						)

										_lowerCAmelCase							=							self.image_processing_class.from_dict(self.image_processor_dict  ,  size=42  ,  crop_size=84						)
										self.assertEqual(image_processor.size  ,  {'shortest_edge': 42}						)
										self.assertEqual(image_processor.crop_size  ,  {'height': 84, 'width': 84}						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										pass





			def   SCREAMING_SNAKE_CASE__     (       self	:	str						):
										# Initialize image_processing
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										# create random PIL images
										_lowerCAmelCase							=							self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__						)
										for image in image_inputs:
																	self.assertIsInstance(lowercase__  ,  Image.Image						)

										# Test not batched input
										_lowerCAmelCase							=							image_processing(image_inputs[0]  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )

										# Test batched
										_lowerCAmelCase							=							image_processing(lowercase__  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )





			def   SCREAMING_SNAKE_CASE__     (       self	:	Any						):
										# Initialize image_processing
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										# create random numpy tensors
										_lowerCAmelCase							=							self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__  ,  numpify=lowercase__						)
										for image in image_inputs:
																	self.assertIsInstance(lowercase__  ,  np.ndarray						)

										# Test not batched input
										_lowerCAmelCase							=							image_processing(image_inputs[0]  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )

										# Test batched
										_lowerCAmelCase							=							image_processing(lowercase__  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )





			def   SCREAMING_SNAKE_CASE__     (       self	:	int						):
										# Initialize image_processing
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										# create random PyTorch tensors
										_lowerCAmelCase							=							self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__  ,  torchify=lowercase__						)
										for image in image_inputs:
																	self.assertIsInstance(lowercase__  ,  torch.Tensor						)

										# Test not batched input
										_lowerCAmelCase							=							image_processing(image_inputs[0]  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )

										# Test batched
										_lowerCAmelCase							=							image_processing(lowercase__  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )







@require_torch
@require_vision
class        lowerCamelCase__				(   UpperCAmelCase       ,unittest.TestCase				):
			UpperCamelCase__    							=ChineseCLIPImageProcessor if is_vision_available() else None





			def   SCREAMING_SNAKE_CASE__     (       self	:	Optional[Any]						):
										_lowerCAmelCase							=							ChineseCLIPImageProcessingTester(self  ,  num_channels=4  ,  do_center_crop=lowercase__						)
										_lowerCAmelCase							=							3





			@property
			def   SCREAMING_SNAKE_CASE__     (       self	:	Tuple						):
										return self.image_processor_tester.prepare_image_processor_dict()





			def   SCREAMING_SNAKE_CASE__     (       self	:	List[Any]						):
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										self.assertTrue(hasattr(lowercase__  ,  'do_resize'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'size'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_center_crop'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'center_crop'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_normalize'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'image_mean'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'image_std'						)						)
										self.assertTrue(hasattr(lowercase__  ,  'do_convert_rgb'						)						)





			def   SCREAMING_SNAKE_CASE__     (       self	:	Union[str, Any]						):
										pass





			def   SCREAMING_SNAKE_CASE__     (       self	:	Dict						):
										# Initialize image_processing
										_lowerCAmelCase							=							self.image_processing_class(**self.image_processor_dict						)
										# create random PIL images
										_lowerCAmelCase							=							self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__						)
										for image in image_inputs:
																	self.assertIsInstance(lowercase__  ,  Image.Image						)

										# Test not batched input
										_lowerCAmelCase							=							image_processing(image_inputs[0]  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        1,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )

										# Test batched
										_lowerCAmelCase							=							image_processing(lowercase__  ,  return_tensors='pt'						).pixel_values
										self.assertEqual(
										    encoded_images.shape  ,  (
										        self.image_processor_tester.batch_size,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )  ,  )






