



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				_lowercase     							=						'''pt'''
elif is_tf_available():
				_lowercase     							=						'''tf'''
else:
				_lowercase     							=						'''jax'''
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Optional[Any]     =			ByTaTokenizer
			UpperCamelCase					:Union[str, Any]     =			False





			def        _snake_case    (self				):
						super().setUp()
						lowerCamelCase__ :      Optional[Any]								=		ByTaTokenizer()
						tokenizer.save_pretrained(self.tmpdirname				)





			@cached_property
			def        _snake_case    (self				):
						return ByTaTokenizer.from_pretrained("""google/byt5-small"""				)





			def        _snake_case    (self							,    **__magic_name__				):
						return self.tokenizer_class.from_pretrained(self.tmpdirname							,    **__magic_name__				)





			def        _snake_case    (self							,    __magic_name__							,    __magic_name__=False							,    __magic_name__=20							,    __magic_name__=5				):
						# XXX The default common tokenizer tests assume that every ID is decodable on its own.
						# This assumption is invalid for ByT5 because single bytes might not be
						# valid utf-8 (byte 128 for instance).
						# Here we're overriding the smallest possible method to provide
						# a clean sequence without making the same assumption.

						lowerCamelCase__ :      Tuple								=		[]
						for i in range(len(__magic_name__				)				):
									try:
												lowerCamelCase__ :      str								=		tokenizer.decode([i]							,    clean_up_tokenization_spaces=__magic_name__				)
									except UnicodeDecodeError:
												pass
									toks.append((i, tok)				)

						lowerCamelCase__ :      Tuple								=		list(filter(lambda __magic_name__				: re.match(R"""^[ a-zA-Z]+$"""							,    t[1]				)							,    __magic_name__				)				)
						lowerCamelCase__ :      Any								=		list(filter(lambda __magic_name__				: [t[0]] == tokenizer.encode(t[1]							,    add_special_tokens=__magic_name__				)							,    __magic_name__				)				)
						if max_length is not None and len(__magic_name__				) > max_length:
									lowerCamelCase__ :      List[Any]								=		toks[:max_length]
						if min_length is not None and len(__magic_name__				) < min_length and len(__magic_name__				) > 0:
									while len(__magic_name__				) < min_length:
												lowerCamelCase__ :      Union[str, Any]								=		toks + toks
        # toks_str = [t[1] for t in toks]
						lowerCamelCase__ :      Optional[Any]								=		[t[0] for t in toks]

						# Ensure consistency
						lowerCamelCase__ :      List[str]								=		tokenizer.decode(__magic_name__							,    clean_up_tokenization_spaces=__magic_name__				)
						if " " not in output_txt and len(__magic_name__				) > 1:
									lowerCamelCase__ :      Any								=		(
									    tokenizer.decode([toks_ids[0]]							,    clean_up_tokenization_spaces=__magic_name__				)
									    + """ """
									    + tokenizer.decode(toks_ids[1:]							,    clean_up_tokenization_spaces=__magic_name__				)
									)
						if with_prefix_space:
									lowerCamelCase__ :      Optional[Any]								=		""" """ + output_txt
						lowerCamelCase__ :      List[Any]								=		tokenizer.encode(__magic_name__							,    add_special_tokens=__magic_name__				)
						return output_txt, output_ids





			def        _snake_case    (self				):
						lowerCamelCase__ :      List[Any]								=		self.ta_base_tokenizer
						lowerCamelCase__ :      Dict								=		tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]				)
						lowerCamelCase__ :      List[str]								=		tokenizer(["""hi""", """I went to the gym""", """"""]				)
						self.assertListEqual(batch_with_eos_added["""input_ids"""]							,    batch_without_eos_added["""input_ids"""]				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      int								=		self.ta_base_tokenizer
						lowerCamelCase__ :      Optional[int]								=		"""Unicode €."""
						lowerCamelCase__ :      Dict								=		tokenizer(__magic_name__				)
						lowerCamelCase__ :      Optional[int]								=		[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
						self.assertEqual(encoded["""input_ids"""]							,    __magic_name__				)

						# decoding
						lowerCamelCase__ :      Optional[int]								=		tokenizer.decode(__magic_name__				)
						self.assertEqual(__magic_name__							,    """Unicode €.</s>"""				)

						lowerCamelCase__ :      Tuple								=		tokenizer("""e è é ê ë"""				)
						lowerCamelCase__ :      Dict								=		[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
						self.assertEqual(encoded["""input_ids"""]							,    __magic_name__				)
						# decoding
						lowerCamelCase__ :      Optional[int]								=		tokenizer.decode(__magic_name__				)
						self.assertEqual(__magic_name__							,    """e è é ê ë</s>"""				)

						# encode/decode, but with `encode` instead of `__call__`
						self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""				)				)							,    """e è é ê ë</s>"""				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Optional[int]								=		self.ta_base_tokenizer
						lowerCamelCase__ :      str								=		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
						# fmt: off
						lowerCamelCase__ :      List[Any]								=		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
						# fmt: on
						lowerCamelCase__ :      Any								=		tokenizer(__magic_name__							,    padding=__magic_name__							,    return_tensors=__magic_name__				)
						self.assertIsInstance(__magic_name__							,    __magic_name__				)

						if FRAMEWORK != "jax":
									lowerCamelCase__ :      Tuple								=		list(batch.input_ids.numpy()[0]				)
						else:
									lowerCamelCase__ :      List[str]								=		list(batch.input_ids.tolist()[0]				)

						self.assertListEqual(__magic_name__							,    __magic_name__				)

						self.assertEqual((2, 37)							,    batch.input_ids.shape				)
						self.assertEqual((2, 37)							,    batch.attention_mask.shape				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Tuple								=		self.ta_base_tokenizer
						lowerCamelCase__ :      Optional[int]								=		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
						lowerCamelCase__ :      List[str]								=		tokenizer(__magic_name__							,    padding=__magic_name__							,    return_tensors=__magic_name__				)
						# check if input_ids are returned and no decoder_input_ids
						self.assertIn("""input_ids"""							,    __magic_name__				)
						self.assertIn("""attention_mask"""							,    __magic_name__				)
						self.assertNotIn("""decoder_input_ids"""							,    __magic_name__				)
						self.assertNotIn("""decoder_attention_mask"""							,    __magic_name__				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Tuple								=		self.ta_base_tokenizer
						lowerCamelCase__ :      List[str]								=		[
						    """Summary of the text.""",
						    """Another summary.""",
						]
						lowerCamelCase__ :      Union[str, Any]								=		tokenizer(
						    text_target=__magic_name__							,    max_length=32							,    padding="""max_length"""							,    truncation=__magic_name__							,    return_tensors=__magic_name__				)
						self.assertEqual(32							,    targets["""input_ids"""].shape[1]				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      str								=		self.ta_base_tokenizer
						lowerCamelCase__ :      Dict								=		["""A long paragraph for summarization. </s>"""]
						lowerCamelCase__ :      Optional[Any]								=		["""Summary of the text. </s>"""]
						# fmt: off
						lowerCamelCase__ :      Tuple								=		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
						lowerCamelCase__ :      Union[str, Any]								=		[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
						# fmt: on

						lowerCamelCase__ :      Optional[int]								=		tokenizer(__magic_name__							,    text_target=__magic_name__				)

						self.assertEqual(__magic_name__							,    batch["""input_ids"""][0]				)
						self.assertEqual(__magic_name__							,    batch["""labels"""][0]				)





			def        _snake_case    (self				):
						# safety check on max_len default value so we are sure the test works
						lowerCamelCase__ :      str								=		self.get_tokenizers()
						for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}"				):
												self.assertNotEqual(tokenizer.model_max_length							,    42				)

        # Now let's start the test
						lowerCamelCase__ :      Any								=		self.get_tokenizers()
						for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}"				):
												# Isolate this from the other tests because we save additional tokens/etc
												lowerCamelCase__ :      List[Any]								=		tempfile.mkdtemp()

												lowerCamelCase__ :      Tuple								=		""" He is very happy, UNwant\u00E9d,running"""
												lowerCamelCase__ :      Dict								=		tokenizer.encode(__magic_name__							,    add_special_tokens=__magic_name__				)
												tokenizer.save_pretrained(__magic_name__				)

												lowerCamelCase__ :      List[Any]								=		tokenizer.__class__.from_pretrained(__magic_name__				)
												lowerCamelCase__ :      Tuple								=		after_tokenizer.encode(__magic_name__							,    add_special_tokens=__magic_name__				)
												self.assertListEqual(__magic_name__							,    __magic_name__				)

												shutil.rmtree(__magic_name__				)

						lowerCamelCase__ :      str								=		self.get_tokenizers(model_max_length=42				)
						for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}"				):
												# Isolate this from the other tests because we save additional tokens/etc
												lowerCamelCase__ :      str								=		tempfile.mkdtemp()

												lowerCamelCase__ :      Tuple								=		""" He is very happy, UNwant\u00E9d,running"""
												tokenizer.add_tokens(["""bim""", """bambam"""]				)
												lowerCamelCase__ :      List[str]								=		tokenizer.additional_special_tokens
												additional_special_tokens.append("""new_additional_special_token"""				)
												tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}				)
												lowerCamelCase__ :      List[Any]								=		tokenizer.encode(__magic_name__							,    add_special_tokens=__magic_name__				)
												tokenizer.save_pretrained(__magic_name__				)

												lowerCamelCase__ :      str								=		tokenizer.__class__.from_pretrained(__magic_name__				)
												lowerCamelCase__ :      str								=		after_tokenizer.encode(__magic_name__							,    add_special_tokens=__magic_name__				)
												self.assertListEqual(__magic_name__							,    __magic_name__				)
												self.assertIn("""new_additional_special_token"""							,    after_tokenizer.additional_special_tokens				)
												self.assertEqual(after_tokenizer.model_max_length							,    42				)

												lowerCamelCase__ :      List[str]								=		tokenizer.__class__.from_pretrained(__magic_name__							,    model_max_length=43				)
												self.assertEqual(tokenizer.model_max_length							,    43				)

												shutil.rmtree(__magic_name__				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      int								=		[]
						if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

						if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer_utils.save_pretrained(__magic_name__				)

												with open(os.path.join(__magic_name__							,    """special_tokens_map.json"""				)							,    encoding="""utf-8"""				) as json_file:
															lowerCamelCase__ :      Optional[Any]								=		json.load(__magic_name__				)

												with open(os.path.join(__magic_name__							,    """tokenizer_config.json"""				)							,    encoding="""utf-8"""				) as json_file:
															lowerCamelCase__ :      Union[str, Any]								=		json.load(__magic_name__				)

												lowerCamelCase__ :      Tuple								=		[f"<extra_id_{i}>" for i in range(125				)]

												lowerCamelCase__ :      Dict								=		added_tokens_extra_ids + [
												    """an_additional_special_token"""
												]
												lowerCamelCase__ :      Any								=		added_tokens_extra_ids + [
												    """an_additional_special_token"""
												]

												with open(os.path.join(__magic_name__							,    """special_tokens_map.json"""				)							,    """w"""							,    encoding="""utf-8"""				) as outfile:
															json.dump(__magic_name__							,    __magic_name__				)
												with open(os.path.join(__magic_name__							,    """tokenizer_config.json"""				)							,    """w"""							,    encoding="""utf-8"""				) as outfile:
															json.dump(__magic_name__							,    __magic_name__				)

												# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
												# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
												# "special_tokens_map.json" files
												lowerCamelCase__ :      Optional[Any]								=		tokenizer_class.from_pretrained(
												    __magic_name__							,    )
												self.assertIn(
												    """an_additional_special_token"""							,    tokenizer_without_change_in_init.additional_special_tokens				)
												# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
												self.assertEqual(
												    ["""an_additional_special_token"""]							,    tokenizer_without_change_in_init.convert_ids_to_tokens(
												        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]				)				)							,    )

												# Now we test that we can change the value of additional_special_tokens in the from_pretrained
												lowerCamelCase__ :      Optional[Any]								=		added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""							,    lstrip=__magic_name__				)]
												lowerCamelCase__ :      Optional[Any]								=		tokenizer_class.from_pretrained(
												    __magic_name__							,    additional_special_tokens=__magic_name__							,    )

												self.assertIn("""a_new_additional_special_token"""							,    tokenizer.additional_special_tokens				)
												self.assertEqual(
												    ["""a_new_additional_special_token"""]							,    tokenizer.convert_ids_to_tokens(
												        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]				)				)							,    )





			def        _snake_case    (self				):
						lowerCamelCase__ :      Optional[int]								=		[]
						if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

						if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer_utils.save_pretrained(__magic_name__				)

												lowerCamelCase__ :      Optional[Any]								=		tokenizer_class.from_pretrained(__magic_name__				)

												self.assertTrue(tokenizer.decode([255]				) == """"""				)





			def        _snake_case    (self				):
						pass





			def        _snake_case    (self				):
						pass





			def        _snake_case    (self				):
						pass





			def        _snake_case    (self				):
						pass





			def        _snake_case    (self				):
						# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
						# and special added tokens as tokens
						lowerCamelCase__ :      Dict								=		self.get_tokenizers(fast=__magic_name__							,    do_lower_case=__magic_name__				)
						for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}"				):
												lowerCamelCase__ :      Dict								=		["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
												lowerCamelCase__ :      Tuple								=		tokenizer.convert_tokens_to_string(__magic_name__				)

												self.assertIsInstance(__magic_name__							,    __magic_name__				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Any								=		self.get_tokenizers()
						for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}"				):
												lowerCamelCase__ :      Optional[Any]								=		[
												    """bos_token""",
												    """eos_token""",
												    """unk_token""",
												    """sep_token""",
												    """pad_token""",
												    """cls_token""",
												    """mask_token""",
												]

												lowerCamelCase__ :      int								=		0
												lowerCamelCase__ :      Any								=		tokenizer.convert_ids_to_tokens(
												    __magic_name__							,    skip_special_tokens=__magic_name__				)

												for attr in attributes_list:
															setattr(__magic_name__							,    attr + """_id"""							,    __magic_name__				)
															self.assertEqual(getattr(__magic_name__							,    __magic_name__				)							,    __magic_name__				)
															self.assertEqual(getattr(__magic_name__							,    attr + """_id"""				)							,    __magic_name__				)

															setattr(__magic_name__							,    attr + """_id"""							,    __magic_name__				)
															self.assertEqual(getattr(__magic_name__							,    __magic_name__				)							,    __magic_name__				)
															self.assertEqual(getattr(__magic_name__							,    attr + """_id"""				)							,    __magic_name__				)

												setattr(__magic_name__							,    """additional_special_tokens_ids"""							,    []				)
												self.assertListEqual(getattr(__magic_name__							,    """additional_special_tokens"""				)							,    []				)
												self.assertListEqual(getattr(__magic_name__							,    """additional_special_tokens_ids"""				)							,    []				)

												setattr(__magic_name__							,    """additional_special_tokens_ids"""							,    [token_id_to_test_setters]				)
												self.assertListEqual(getattr(__magic_name__							,    """additional_special_tokens"""				)							,    [token_to_test_setters]				)
												self.assertListEqual(getattr(__magic_name__							,    """additional_special_tokens_ids"""				)							,    [token_id_to_test_setters]				)








import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


_lowercase     							=						logging.get_logger(__name__)






def      _A				(UpperCamelCase		:      Dict      ,			UpperCamelCase		:      Optional[int] )  ->Tuple:



			'''simple docstring'''







			lowerCamelCase__ :      List[str]								=		nn.functional.normalize(UpperCamelCase )
			lowerCamelCase__ :      Tuple								=		nn.functional.normalize(UpperCamelCase )
			return torch.mm(UpperCamelCase      ,			normalized_text_embeds.t() )
class 				__A					( A_      ):
			UpperCamelCase					:Optional[Any]     =			CLIPConfig

			UpperCamelCase					:Optional[Any]     =			['''CLIPEncoderLayer''']





			def __init__(self							,    __magic_name__				):
						super().__init__(__magic_name__				)

						lowerCamelCase__ :      Union[str, Any]								=		CLIPVisionModel(config.vision_config				)
						lowerCamelCase__ :      Any								=		nn.Linear(config.vision_config.hidden_size							,    config.projection_dim							,    bias=__magic_name__				)

						lowerCamelCase__ :      Any								=		nn.Parameter(torch.ones(17							,    config.projection_dim				)							,    requires_grad=__magic_name__				)
						lowerCamelCase__ :      Union[str, Any]								=		nn.Parameter(torch.ones(3							,    config.projection_dim				)							,    requires_grad=__magic_name__				)

						lowerCamelCase__ :      Any								=		nn.Parameter(torch.ones(17				)							,    requires_grad=__magic_name__				)
						lowerCamelCase__ :      Optional[Any]								=		nn.Parameter(torch.ones(3				)							,    requires_grad=__magic_name__				)





			@torch.no_grad()
			def        _snake_case    (self							,    __magic_name__							,    __magic_name__				):
						lowerCamelCase__ :      Any								=		self.vision_model(__magic_name__				)[1]  # pooled_output
						lowerCamelCase__ :      Tuple								=		self.visual_projection(__magic_name__				)

						# we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
						lowerCamelCase__ :      Any								=		cosine_distance(__magic_name__							,    self.special_care_embeds				).cpu().float().numpy()
						lowerCamelCase__ :      str								=		cosine_distance(__magic_name__							,    self.concept_embeds				).cpu().float().numpy()

						lowerCamelCase__ :      int								=		[]
						lowerCamelCase__ :      str								=		image_embeds.shape[0]
						for i in range(__magic_name__				):
									lowerCamelCase__ :      str								=		{"""special_scores""": {}, """special_care""": [], """concept_scores""": {}, """bad_concepts""": []}

									# increase this value to create a stronger `nfsw` filter
									# at the cost of increasing the possibility of filtering benign images
									lowerCamelCase__ :      Optional[Any]								=		0.0

									for concept_idx in range(len(special_cos_dist[0]				)				):
												lowerCamelCase__ :      List[str]								=		special_cos_dist[i][concept_idx]
												lowerCamelCase__ :      Tuple								=		self.special_care_embeds_weights[concept_idx].item()
												lowerCamelCase__ :      str								=		round(concept_cos - concept_threshold + adjustment							,    3				)
												if result_img["special_scores"][concept_idx] > 0:
															result_img["special_care"].append({concept_idx, result_img["""special_scores"""][concept_idx]}				)
															lowerCamelCase__ :      List[str]								=		0.01

									for concept_idx in range(len(cos_dist[0]				)				):
												lowerCamelCase__ :      str								=		cos_dist[i][concept_idx]
												lowerCamelCase__ :      Dict								=		self.concept_embeds_weights[concept_idx].item()
												lowerCamelCase__ :      int								=		round(concept_cos - concept_threshold + adjustment							,    3				)
												if result_img["concept_scores"][concept_idx] > 0:
															result_img["bad_concepts"].append(__magic_name__				)

									result.append(__magic_name__				)

						lowerCamelCase__ :      Tuple								=		[len(res["""bad_concepts"""]				) > 0 for res in result]

						return images, has_nsfw_concepts





			@torch.no_grad()
			def        _snake_case    (self							,    __magic_name__							,    __magic_name__				):
						lowerCamelCase__ :      Union[str, Any]								=		self.vision_model(__magic_name__				)[1]  # pooled_output
						lowerCamelCase__ :      str								=		self.visual_projection(__magic_name__				)

						lowerCamelCase__ :      Optional[Any]								=		cosine_distance(__magic_name__							,    self.special_care_embeds				)
						lowerCamelCase__ :      Dict								=		cosine_distance(__magic_name__							,    self.concept_embeds				)

						# increase this value to create a stronger `nsfw` filter
						# at the cost of increasing the possibility of filtering benign images
						lowerCamelCase__ :      Any								=		0.0

						lowerCamelCase__ :      Dict								=		special_cos_dist - self.special_care_embeds_weights + adjustment
						# special_scores = special_scores.round(decimals=3)
						lowerCamelCase__ :      Union[str, Any]								=		torch.any(special_scores > 0							,    dim=1				)
						lowerCamelCase__ :      Optional[Any]								=		special_care * 0.01
						lowerCamelCase__ :      Dict								=		special_adjustment.unsqueeze(1				).expand(-1							,    cos_dist.shape[1]				)

						lowerCamelCase__ :      Tuple								=		(cos_dist - self.concept_embeds_weights) + special_adjustment
						# concept_scores = concept_scores.round(decimals=3)
						lowerCamelCase__ :      Any								=		torch.any(concept_scores > 0							,    dim=1				)

						return images, has_nsfw_concepts



