




"""simple docstring"""





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import ChineseCLIPImageProcessor



class   a__					(						unittest.TestCase ):


			def __init__(   self   :     List[str]   ,	UpperCamelCase_   :     Dict   ,	UpperCamelCase_   :     Optional[int]=7   ,	UpperCamelCase_   :     str=3   ,	UpperCamelCase_   :     Union[str, Any]=18   ,	UpperCamelCase_   :     Union[str, Any]=30   ,	UpperCamelCase_   :     List[str]=400   ,	UpperCamelCase_   :     int=True   ,	UpperCamelCase_   :     Optional[int]=None   ,	UpperCamelCase_   :     Tuple=True   ,	UpperCamelCase_   :     Any=None   ,	UpperCamelCase_   :     Optional[int]=True   ,	UpperCamelCase_   :     int=[0.48145466, 0.4578275, 0.40821073]   ,	UpperCamelCase_   :     str=[0.26862954, 0.26130258, 0.27577711]   ,	UpperCamelCase_   :     List[Any]=True   ,	):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       size if size is not None else {'height': 224, 'width': 224}
						__UpperCAmelCase						:			Optional[Any]												=       crop_size if crop_size is not None else {'height': 18, 'width': 18}
						__UpperCAmelCase						:			Tuple												=       parent
						__UpperCAmelCase						:			int												=       batch_size
						__UpperCAmelCase						:			Dict												=       num_channels
						__UpperCAmelCase						:			Any												=       image_size
						__UpperCAmelCase						:			Dict												=       min_resolution
						__UpperCAmelCase						:			List[Any]												=       max_resolution
						__UpperCAmelCase						:			str												=       do_resize
						__UpperCAmelCase						:			Union[str, Any]												=       size
						__UpperCAmelCase						:			Any												=       do_center_crop
						__UpperCAmelCase						:			List[str]												=       crop_size
						__UpperCAmelCase						:			Union[str, Any]												=       do_normalize
						__UpperCAmelCase						:			Optional[int]												=       image_mean
						__UpperCAmelCase						:			int												=       image_std
						__UpperCAmelCase						:			List[str]												=       do_convert_rgb


			def    a_  (   self   :     Union[str, Any]):


						"""simple docstring"""




						return {
						    "do_resize": self.do_resize,
						    "size": self.size,
						    "do_center_crop": self.do_center_crop,
						    "crop_size": self.crop_size,
						    "do_normalize": self.do_normalize,
						    "image_mean": self.image_mean,
						    "image_std": self.image_std,
						    "do_convert_rgb": self.do_convert_rgb,
						}


			def    a_  (   self   :     Tuple   ,	UpperCamelCase_   :     Any=False   ,	UpperCamelCase_   :     Union[str, Any]=False   ,	UpperCamelCase_   :     Tuple=False):


						"""simple docstring"""




						assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

						if equal_resolution:
									__UpperCAmelCase						:			Dict												=       []
									for i in range(self.batch_size):
												image_inputs.append(
												    np.random.randint(
												        255   ,	size=(self.num_channels, self.max_resolution, self.max_resolution)   ,	dtype=np.uinta))
						else:
									__UpperCAmelCase						:			int												=       []
									for i in range(self.batch_size):
												__UpperCAmelCase						:			Tuple												=       np.random.choice(np.arange(self.min_resolution   ,	self.max_resolution)   ,	2)
												image_inputs.append(np.random.randint(255   ,	size=(self.num_channels, width, height)   ,	dtype=np.uinta))

						if not numpify and not torchify:
									# PIL expects the channel dimension as last dimension
									__UpperCAmelCase						:			int												=       [Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE_   ,	0   ,	-1)) for x in image_inputs]

						if torchify:
									__UpperCAmelCase						:			Dict												=       [torch.from_numpy(SCREAMING_SNAKE_CASE_) for x in image_inputs]

						return image_inputs






@require_torch
@require_vision
class   a__					(						_UpperCAmelCase				,     unittest.TestCase ):
			lowercase_		=						ChineseCLIPImageProcessor if is_vision_available() else None


			def    a_  (   self   :     List[Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			Any												=       ChineseCLIPImageProcessingTester(self   ,	do_center_crop=SCREAMING_SNAKE_CASE_)


			@property
			def    a_  (   self   :     Optional[int]):


						"""simple docstring"""




						return self.image_processor_tester.prepare_image_processor_dict()


			def    a_  (   self   :     List[str]):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       self.image_processing_class(**self.image_processor_dict)
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_resize"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"size"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_center_crop"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"center_crop"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_normalize"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"image_mean"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"image_std"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_convert_rgb"))


			def    a_  (   self   :     str):


						"""simple docstring"""




						__UpperCAmelCase						:			List[str]												=       self.image_processing_class.from_dict(self.image_processor_dict)
						self.assertEqual(image_processor.size   ,	{"height": 224, "width": 224})
						self.assertEqual(image_processor.crop_size   ,	{"height": 18, "width": 18})

						__UpperCAmelCase						:			str												=       self.image_processing_class.from_dict(self.image_processor_dict   ,	size=42   ,	crop_size=84)
						self.assertEqual(image_processor.size   ,	{"shortest_edge": 42})
						self.assertEqual(image_processor.crop_size   ,	{"height": 84, "width": 84})


			def    a_  (   self   :     List[Any]):


						"""simple docstring"""




						pass


			def    a_  (   self   :     Optional[Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       self.image_processing_class(**self.image_processor_dict)
						# create random PIL images
						__UpperCAmelCase						:			Any												=       self.image_processor_tester.prepare_inputs(equal_resolution=SCREAMING_SNAKE_CASE_)
						for image in image_inputs:
									self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,	Image.Image)

						# Test not batched input
						__UpperCAmelCase						:			Dict												=       image_processing(image_inputs[0]   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)

						# Test batched
						__UpperCAmelCase						:			Tuple												=       image_processing(SCREAMING_SNAKE_CASE_   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)


			def    a_  (   self   :     Optional[int]):


						"""simple docstring"""




						__UpperCAmelCase						:			Any												=       self.image_processing_class(**self.image_processor_dict)
						# create random numpy tensors
						__UpperCAmelCase						:			Optional[int]												=       self.image_processor_tester.prepare_inputs(equal_resolution=SCREAMING_SNAKE_CASE_   ,	numpify=SCREAMING_SNAKE_CASE_)
						for image in image_inputs:
									self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,	np.ndarray)

						# Test not batched input
						__UpperCAmelCase						:			Dict												=       image_processing(image_inputs[0]   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)

						# Test batched
						__UpperCAmelCase						:			int												=       image_processing(SCREAMING_SNAKE_CASE_   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)


			def    a_  (   self   :     Optional[Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			Optional[int]												=       self.image_processing_class(**self.image_processor_dict)
						# create random PyTorch tensors
						__UpperCAmelCase						:			Optional[int]												=       self.image_processor_tester.prepare_inputs(equal_resolution=SCREAMING_SNAKE_CASE_   ,	torchify=SCREAMING_SNAKE_CASE_)
						for image in image_inputs:
									self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,	torch.Tensor)

						# Test not batched input
						__UpperCAmelCase						:			Optional[int]												=       image_processing(image_inputs[0]   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        1,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)

						# Test batched
						__UpperCAmelCase						:			List[str]												=       image_processing(SCREAMING_SNAKE_CASE_   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)






@require_torch
@require_vision
class   a__					(						_UpperCAmelCase				,     unittest.TestCase ):
			lowercase_		=						ChineseCLIPImageProcessor if is_vision_available() else None


			def    a_  (   self   :     List[Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			str												=       ChineseCLIPImageProcessingTester(self   ,	num_channels=4   ,	do_center_crop=SCREAMING_SNAKE_CASE_)
						__UpperCAmelCase						:			List[str]												=       3


			@property
			def    a_  (   self   :     Union[str, Any]):


						"""simple docstring"""




						return self.image_processor_tester.prepare_image_processor_dict()


			def    a_  (   self   :     str):


						"""simple docstring"""




						__UpperCAmelCase						:			List[str]												=       self.image_processing_class(**self.image_processor_dict)
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_resize"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"size"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_center_crop"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"center_crop"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_normalize"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"image_mean"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"image_std"))
						self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_   ,	"do_convert_rgb"))


			def    a_  (   self   :     Optional[Any]):


						"""simple docstring"""




						pass


			def    a_  (   self   :     List[str]):


						"""simple docstring"""




						__UpperCAmelCase						:			Any												=       self.image_processing_class(**self.image_processor_dict)
						# create random PIL images
						__UpperCAmelCase						:			Dict												=       self.image_processor_tester.prepare_inputs(equal_resolution=SCREAMING_SNAKE_CASE_)
						for image in image_inputs:
									self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,	Image.Image)

						# Test not batched input
						__UpperCAmelCase						:			Any												=       image_processing(image_inputs[0]   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        1,
						        self.expected_encoded_image_num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)

						# Test batched
						__UpperCAmelCase						:			List[Any]												=       image_processing(SCREAMING_SNAKE_CASE_   ,	return_tensors="pt").pixel_values
						self.assertEqual(
						    encoded_images.shape   ,	(
						        self.image_processor_tester.batch_size,
						        self.expected_encoded_image_num_channels,
						        self.image_processor_tester.crop_size["height"],
						        self.image_processor_tester.crop_size["width"],
						    )   ,	)






'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A__						:   str				=    logging.get_logger(__name__)

A__						:   str				=    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

A__						:   Tuple				=    {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

A__						:   str				=    {
    """junnyu/roformer_chinese_small""": 1536,
    """junnyu/roformer_chinese_base""": 1536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


A__						:   Tuple				=    {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    lowerCamelCase    :   Union[str, Any]					=     VOCAB_FILES_NAMES
    lowerCamelCase    :   int					=     PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase    :   Any					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase    :   Optional[int]					=     PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase    :   Dict					=     RoFormerTokenizer







    def __init__(				self     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_="[UNK]"     ,	SCREAMING_SNAKE_CASE_="[SEP]"     ,	SCREAMING_SNAKE_CASE_="[PAD]"     ,	SCREAMING_SNAKE_CASE_="[CLS]"     ,	SCREAMING_SNAKE_CASE_="[MASK]"     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Optional[Any]:
         super().__init__(
             SCREAMING_SNAKE_CASE_     ,	tokenizer_file=SCREAMING_SNAKE_CASE_     ,	do_lower_case=SCREAMING_SNAKE_CASE_     ,	unk_token=SCREAMING_SNAKE_CASE_     ,	sep_token=SCREAMING_SNAKE_CASE_     ,	pad_token=SCREAMING_SNAKE_CASE_     ,	cls_token=SCREAMING_SNAKE_CASE_     ,	mask_token=SCREAMING_SNAKE_CASE_     ,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE_     ,	strip_accents=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)

         __lowerCamelCase  :      Optional[int]														=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
         if (
             pre_tok_state.get('lowercase'     ,	SCREAMING_SNAKE_CASE_      ) != do_lower_case
             or pre_tok_state.get('strip_accents'     ,	SCREAMING_SNAKE_CASE_      ) != strip_accents
         ):
              __lowerCamelCase  :      Optional[int]														=						getattr(SCREAMING_SNAKE_CASE_     ,	pre_tok_state.pop('type'      )      )
              __lowerCamelCase  :      Union[str, Any]														=						do_lower_case
              __lowerCamelCase  :      str														=						strip_accents
              __lowerCamelCase  :      Optional[Any]														=						pre_tok_class(**SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      Tuple														=						do_lower_case







    def __getstate__(				self      )							->	List[str]:
         __lowerCamelCase  :      Union[str, Any]														=						self.__dict__.copy()
         __lowerCamelCase  :      Dict														=						BertPreTokenizer()
         return state







    def __setstate__(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Optional[Any]:
         __lowerCamelCase  :      Optional[int]														=						d
         __lowerCamelCase  :      List[Any]														=						self.__dict__['_tokenizer'].get_vocab()
         __lowerCamelCase  :      Union[str, Any]														=						PreTokenizer.custom(JiebaPreTokenizer(SCREAMING_SNAKE_CASE_      )      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=None      )							->	str:
         __lowerCamelCase  :      Union[str, Any]														=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	List[int]:
         __lowerCamelCase  :      List[str]														=						[self.sep_token_id]
         __lowerCamelCase  :      Dict														=						[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	Tuple[str]:
         __lowerCamelCase  :      Optional[Any]														=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE_     ,	name=SCREAMING_SNAKE_CASE_      )
         return tuple(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=False     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Any:
         __lowerCamelCase  :      Tuple														=						BertPreTokenizer()
         return super().save_pretrained(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )






