


'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import OwlViTImageProcessor, OwlViTProcessor


@require_vision
class 						_snake_case   (       unittest.TestCase		):




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Union[str, Any]       				=       tempfile.mkdtemp()

								# fmt: off
								UpperCAmelCase__					:   int       				=       ["""""", """l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								UpperCAmelCase__					:   List[Any]       				=       dict(zip(_lowerCamelCase							,							range(len(_lowerCamelCase))))
								UpperCAmelCase__					:   Dict       				=       ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
								UpperCAmelCase__					:   int       				=       {"""unk_token""": """<unk>"""}

								UpperCAmelCase__					:   str       				=       os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""vocab_file"""])
								UpperCAmelCase__					:   Dict       				=       os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""merges_file"""])
								with open(self.vocab_file							,							"""w"""							,							encoding="""utf-8""") as fp:
											fp.write(json.dumps(_lowerCamelCase) + """\n""")
								with open(self.merges_file							,							"""w"""							,							encoding="""utf-8""") as fp:
											fp.write("""\n""".join(_lowerCamelCase))

								UpperCAmelCase__					:   Optional[int]       				=       {
								    """do_resize""": True,
								    """size""": 20,
								    """do_center_crop""": True,
								    """crop_size""": 18,
								    """do_normalize""": True,
								    """image_mean""": [0.48145466, 0.4578275, 0.40821073],
								    """image_std""": [0.26862954, 0.26130258, 0.27577711],
								}
								UpperCAmelCase__					:   str       				=       os.path.join(self.tmpdirname							,							_lowerCamelCase)
								with open(self.image_processor_file							,							"""w"""							,							encoding="""utf-8""") as fp:
											json.dump(_lowerCamelCase							,							_lowerCamelCase)




					def 			snake_case__  ( self							,							**_lowerCamelCase):
								return CLIPTokenizer.from_pretrained(self.tmpdirname							,							pad_token="""!"""							,							**_lowerCamelCase)




					def 			snake_case__  ( self							,							**_lowerCamelCase):
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,							pad_token="""!"""							,							**_lowerCamelCase)




					def 			snake_case__  ( self							,							**_lowerCamelCase):
								return OwlViTImageProcessor.from_pretrained(self.tmpdirname							,							**_lowerCamelCase)




					def 			snake_case__  ( self):
								shutil.rmtree(self.tmpdirname)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Dict       				=       [np.random.randint(255							,							size=(3, 30, 400)							,							dtype=np.uinta)]

								UpperCAmelCase__					:   Tuple       				=       [Image.fromarray(np.moveaxis(_lowerCamelCase							,							0							,							-1)) for x in image_inputs]

								return image_inputs




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Tuple       				=       self.get_tokenizer()
								UpperCAmelCase__					:   Any       				=       self.get_rust_tokenizer()
								UpperCAmelCase__					:   str       				=       self.get_image_processor()

								UpperCAmelCase__					:   Dict       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)
								processor_slow.save_pretrained(self.tmpdirname)
								UpperCAmelCase__					:   List[Any]       				=       OwlViTProcessor.from_pretrained(self.tmpdirname							,							use_fast=_lowerCamelCase)

								UpperCAmelCase__					:   Union[str, Any]       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)
								processor_fast.save_pretrained(self.tmpdirname)
								UpperCAmelCase__					:   Optional[Any]       				=       OwlViTProcessor.from_pretrained(self.tmpdirname)

								self.assertEqual(processor_slow.tokenizer.get_vocab()							,							tokenizer_slow.get_vocab())
								self.assertEqual(processor_fast.tokenizer.get_vocab()							,							tokenizer_fast.get_vocab())
								self.assertEqual(tokenizer_slow.get_vocab()							,							tokenizer_fast.get_vocab())
								self.assertIsInstance(processor_slow.tokenizer							,							_lowerCamelCase)
								self.assertIsInstance(processor_fast.tokenizer							,							_lowerCamelCase)

								self.assertEqual(processor_slow.image_processor.to_json_string()							,							image_processor.to_json_string())
								self.assertEqual(processor_fast.image_processor.to_json_string()							,							image_processor.to_json_string())
								self.assertIsInstance(processor_slow.image_processor							,							_lowerCamelCase)
								self.assertIsInstance(processor_fast.image_processor							,							_lowerCamelCase)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[int]       				=       OwlViTProcessor(tokenizer=self.get_tokenizer()							,							image_processor=self.get_image_processor())
								processor.save_pretrained(self.tmpdirname)

								UpperCAmelCase__					:   Any       				=       self.get_tokenizer(bos_token="""(BOS)"""							,							eos_token="""(EOS)""")
								UpperCAmelCase__					:   str       				=       self.get_image_processor(do_normalize=_lowerCamelCase)

								UpperCAmelCase__					:   List[str]       				=       OwlViTProcessor.from_pretrained(
								    self.tmpdirname							,							bos_token="""(BOS)"""							,							eos_token="""(EOS)"""							,							do_normalize=_lowerCamelCase)

								self.assertEqual(processor.tokenizer.get_vocab()							,							tokenizer_add_kwargs.get_vocab())
								self.assertIsInstance(processor.tokenizer							,							_lowerCamelCase)

								self.assertEqual(processor.image_processor.to_json_string()							,							image_processor_add_kwargs.to_json_string())
								self.assertIsInstance(processor.image_processor							,							_lowerCamelCase)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   List[Any]       				=       self.get_image_processor()
								UpperCAmelCase__					:   str       				=       self.get_tokenizer()

								UpperCAmelCase__					:   Dict       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)

								UpperCAmelCase__					:   Dict       				=       self.prepare_image_inputs()

								UpperCAmelCase__					:   List[Any]       				=       image_processor(_lowerCamelCase							,							return_tensors="""np""")
								UpperCAmelCase__					:   List[Any]       				=       processor(images=_lowerCamelCase							,							return_tensors="""np""")

								for key in input_image_proc.keys():
											self.assertAlmostEqual(input_image_proc[key].sum()							,							input_processor[key].sum()							,							delta=1e-2)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   int       				=       self.get_image_processor()
								UpperCAmelCase__					:   Union[str, Any]       				=       self.get_tokenizer()

								UpperCAmelCase__					:   Optional[int]       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)

								UpperCAmelCase__					:   List[str]       				=       """lower newer"""

								UpperCAmelCase__					:   int       				=       processor(text=_lowerCamelCase							,							return_tensors="""np""")

								UpperCAmelCase__					:   List[str]       				=       tokenizer(_lowerCamelCase							,							return_tensors="""np""")

								for key in encoded_tok.keys():
											self.assertListEqual(encoded_tok[key][0].tolist()							,							encoded_processor[key][0].tolist())




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   int       				=       self.get_image_processor()
								UpperCAmelCase__					:   Optional[Any]       				=       self.get_tokenizer()

								UpperCAmelCase__					:   str       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)

								UpperCAmelCase__					:   Optional[int]       				=       """lower newer"""
								UpperCAmelCase__					:   Union[str, Any]       				=       self.prepare_image_inputs()

								UpperCAmelCase__					:   Union[str, Any]       				=       processor(text=_lowerCamelCase							,							images=_lowerCamelCase)

								self.assertListEqual(list(inputs.keys())							,							["""input_ids""", """attention_mask""", """pixel_values"""])

								# test if it raises when no input is passed
								with pytest.raises(_lowerCamelCase):
											processor()




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       """google/owlvit-base-patch32"""
								UpperCAmelCase__					:   Tuple       				=       OwlViTProcessor.from_pretrained(_lowerCamelCase)

								UpperCAmelCase__					:   Dict       				=       ["""cat""", """nasa badge"""]
								UpperCAmelCase__					:   Union[str, Any]       				=       processor(text=_lowerCamelCase)

								UpperCAmelCase__					:   Any       				=       16
								self.assertListEqual(list(inputs.keys())							,							["""input_ids""", """attention_mask"""])
								self.assertEqual(inputs["""input_ids"""].shape							,							(2, seq_length))

								# test if it raises when no input is passed
								with pytest.raises(_lowerCamelCase):
											processor()




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Union[str, Any]       				=       """google/owlvit-base-patch32"""
								UpperCAmelCase__					:   Tuple       				=       OwlViTProcessor.from_pretrained(_lowerCamelCase)

								UpperCAmelCase__					:   Optional[Any]       				=       [["""cat""", """nasa badge"""], ["""person"""]]
								UpperCAmelCase__					:   List[str]       				=       processor(text=_lowerCamelCase)

								UpperCAmelCase__					:   int       				=       16
								UpperCAmelCase__					:   Dict       				=       len(_lowerCamelCase)
								UpperCAmelCase__					:   Union[str, Any]       				=       max([len(_lowerCamelCase) for texts in input_texts])

								self.assertListEqual(list(inputs.keys())							,							["""input_ids""", """attention_mask"""])
								self.assertEqual(inputs["""input_ids"""].shape							,							(batch_size * num_max_text_queries, seq_length))

								# test if it raises when no input is passed
								with pytest.raises(_lowerCamelCase):
											processor()




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Dict       				=       """google/owlvit-base-patch32"""
								UpperCAmelCase__					:   List[Any]       				=       OwlViTProcessor.from_pretrained(_lowerCamelCase)

								UpperCAmelCase__					:   Tuple       				=       ["""cat""", """nasa badge"""]
								UpperCAmelCase__					:   Any       				=       processor(text=_lowerCamelCase)

								UpperCAmelCase__					:   Optional[int]       				=       16
								UpperCAmelCase__					:   List[str]       				=       inputs["""input_ids"""]
								UpperCAmelCase__					:   Tuple       				=       [
								    [4_9406, 2368, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								    [4_9406, 6841, 1_1301, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
								]

								self.assertListEqual(list(inputs.keys())							,							["""input_ids""", """attention_mask"""])
								self.assertEqual(inputs["""input_ids"""].shape							,							(2, seq_length))
								self.assertListEqual(list(input_ids[0])							,							predicted_ids[0])
								self.assertListEqual(list(input_ids[1])							,							predicted_ids[1])




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   str       				=       self.get_image_processor()
								UpperCAmelCase__					:   List[str]       				=       self.get_tokenizer()

								UpperCAmelCase__					:   Optional[Any]       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)

								UpperCAmelCase__					:   str       				=       self.prepare_image_inputs()
								UpperCAmelCase__					:   Any       				=       self.prepare_image_inputs()

								UpperCAmelCase__					:   List[str]       				=       processor(images=_lowerCamelCase							,							query_images=_lowerCamelCase)

								self.assertListEqual(list(inputs.keys())							,							["""query_pixel_values""", """pixel_values"""])

								# test if it raises when no input is passed
								with pytest.raises(_lowerCamelCase):
											processor()




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[Any]       				=       self.get_image_processor()
								UpperCAmelCase__					:   Union[str, Any]       				=       self.get_tokenizer()

								UpperCAmelCase__					:   List[str]       				=       OwlViTProcessor(tokenizer=_lowerCamelCase							,							image_processor=_lowerCamelCase)

								UpperCAmelCase__					:   Optional[int]       				=       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

								UpperCAmelCase__					:   Dict       				=       processor.batch_decode(_lowerCamelCase)
								UpperCAmelCase__					:   Union[str, Any]       				=       tokenizer.batch_decode(_lowerCamelCase)

								self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)



'''simple docstring'''



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__A		=logging.get_logger(__name__)

__A		={'vocab_file': 'spiece.model'}

__A		={
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}


class 						_snake_case   (       a__		):




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase=False							,							_lowerCamelCase=True							,							_lowerCamelCase=False							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="<unk>"							,							_lowerCamelCase="<sep>"							,							_lowerCamelCase="<pad>"							,							_lowerCamelCase="<cls>"							,							_lowerCamelCase="<mask>"							,							_lowerCamelCase=["<eop>", "<eod>"]							,							_lowerCamelCase = None							,							**_lowerCamelCase							,							):
								UpperCAmelCase__					:   Dict       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else mask_token

								UpperCAmelCase__					:   Any       				=       {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=_lowerCamelCase							,							remove_space=_lowerCamelCase							,							keep_accents=_lowerCamelCase							,							bos_token=_lowerCamelCase							,							eos_token=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							additional_special_tokens=_lowerCamelCase							,							sp_model_kwargs=self.sp_model_kwargs							,							**_lowerCamelCase							,							)

								UpperCAmelCase__					:   List[str]       				=       3

								UpperCAmelCase__					:   str       				=       do_lower_case
								UpperCAmelCase__					:   Tuple       				=       remove_space
								UpperCAmelCase__					:   List[str]       				=       keep_accents
								UpperCAmelCase__					:   Any       				=       vocab_file

								UpperCAmelCase__					:   str       				=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
								self.sp_model.Load(_lowerCamelCase)

								try:
											import jieba
								except ModuleNotFoundError as error:
											raise error.__class__(
											    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
											    """See https://pypi.org/project/jieba/ for installation.""")
								UpperCAmelCase__					:   Union[str, Any]       				=       jieba
								UpperCAmelCase__					:   int       				=       str.maketrans(""" \n"""							,							"""\u2582\u2583""")




					@property
					# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
					def 			snake_case__  ( self):
								return len(self.sp_model)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[Any]       				=       {self.convert_ids_to_tokens(_lowerCamelCase): i for i in range(self.vocab_size)}
								vocab.update(self.added_tokens_encoder)
								return vocab




					def __getstate__( self):
								UpperCAmelCase__					:   List[str]       				=       self.__dict__.copy()
								UpperCAmelCase__					:   Optional[int]       				=       None
								return state




					def __setstate__( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   Optional[Any]       				=       d

								# for backward compatibility
								if not hasattr(self							,							"""sp_model_kwargs"""):
											UpperCAmelCase__					:   Tuple       				=       {}

								UpperCAmelCase__					:   Union[str, Any]       				=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
								self.sp_model.Load(self.vocab_file)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								if self.remove_space:
											UpperCAmelCase__					:   Tuple       				=       """ """.join(inputs.strip().split())
								else:
											UpperCAmelCase__					:   str       				=       inputs
								UpperCAmelCase__					:   List[Any]       				=       outputs.replace("""``"""							,							"""\"""").replace("""''"""							,							"""\"""")

								if not self.keep_accents:
											UpperCAmelCase__					:   str       				=       unicodedata.normalize("""NFKD"""							,							_lowerCamelCase)
											UpperCAmelCase__					:   Any       				=       """""".join([c for c in outputs if not unicodedata.combining(_lowerCamelCase)])
								if self.do_lower_case:
											UpperCAmelCase__					:   Optional[int]       				=       outputs.lower()

								return outputs




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   List[str]       				=       self.preprocess_text(_lowerCamelCase)
								UpperCAmelCase__					:   Dict       				=       self.sp_model.encode(_lowerCamelCase							,							out_type=_lowerCamelCase)
								UpperCAmelCase__					:   List[Any]       				=       []
								for piece in pieces:
											if len(_lowerCamelCase) > 1 and piece[-1] == str(""",""") and piece[-2].isdigit():
														UpperCAmelCase__					:   str       				=       self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCamelCase							,							""""""))
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																	if len(cur_pieces[0]) == 1:
																				UpperCAmelCase__					:   Optional[int]       				=       cur_pieces[1:]
																	else:
																				UpperCAmelCase__					:   Optional[int]       				=       cur_pieces[0][1:]
														cur_pieces.append(piece[-1])
														new_pieces.extend(_lowerCamelCase)
											else:
														new_pieces.append(_lowerCamelCase)

								return new_pieces




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.sp_model.PieceToId(_lowerCamelCase)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.sp_model.IdToPiece(_lowerCamelCase)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   str       				=       """""".join(_lowerCamelCase).replace(_lowerCamelCase							,							""" """).strip()
								return out_string




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__					:   Optional[Any]       				=       [self.sep_token_id]
								UpperCAmelCase__					:   Optional[int]       				=       [self.cls_token_id]
								if token_ids_a is None:
											return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = False):

								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_lowerCamelCase							,							token_ids_a=_lowerCamelCase							,							already_has_special_tokens=_lowerCamelCase)

								if token_ids_a is not None:
											return ([0] * len(_lowerCamelCase)) + [1] + ([0] * len(_lowerCamelCase)) + [1, 1]
								return ([0] * len(_lowerCamelCase)) + [1, 1]




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__					:   int       				=       [self.sep_token_id]
								UpperCAmelCase__					:   Tuple       				=       [2]

								if token_ids_a is None:
											return len(token_ids_a + sep) * [0] + cls_segment_id
								return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not os.path.isdir(_lowerCamelCase):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
											return
								UpperCAmelCase__					:   str       				=       os.path.join(
								    _lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

								if os.path.abspath(self.vocab_file) != os.path.abspath(_lowerCamelCase) and os.path.isfile(self.vocab_file):
											copyfile(self.vocab_file							,							_lowerCamelCase)
								elif not os.path.isfile(self.vocab_file):
											with open(_lowerCamelCase							,							"""wb""") as fi:
														UpperCAmelCase__					:   str       				=       self.sp_model.serialized_model_proto()
														fi.write(_lowerCamelCase)

								return (out_vocab_file,)




					def 			snake_case__  ( self							,							*_lowerCamelCase							,							**_lowerCamelCase):
								UpperCAmelCase__					:   Union[str, Any]       				=       super()._decode(*_lowerCamelCase							,							**_lowerCamelCase)
								UpperCAmelCase__					:   Any       				=       text.replace(""" """							,							"""""").replace("""\u2582"""							,							""" """).replace("""\u2583"""							,							"""\n""")
								return text