





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__           =			get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
					from transformers.models.mbart.modeling_mbart import shift_tokens_right

UpperCamelCase__           =			250_004
UpperCamelCase__           =			250_020

@require_sentencepiece
@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (  snake_case_ ,		unittest.TestCase							):
							snake_case  :   Tuple      =							MBartTokenizer
							snake_case  :   Any      =							MBartTokenizerFast
							snake_case  :   List[Any]      =							True
							snake_case  :   List[Any]      =							True






							def 					_lowerCamelCase		(    self	):
														super().setUp()

														# We have a SentencePiece fixture for testing
														UpperCamelCase__					  =      MBartTokenizer(__lowerCAmelCase	,			keep_accents=__lowerCAmelCase	)
														tokenizer.save_pretrained(self.tmpdirname	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      MBartTokenizer(__lowerCAmelCase	,			keep_accents=__lowerCAmelCase	)

														UpperCamelCase__					  =      tokenizer.tokenize("""This is a test"""	)
														self.assertListEqual(__lowerCAmelCase	,			["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,			)

														UpperCamelCase__					  =      tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
														self.assertListEqual(
														    __lowerCAmelCase	,			[
														        SPIECE_UNDERLINE + """I""",
														        SPIECE_UNDERLINE + """was""",
														        SPIECE_UNDERLINE + """b""",
														        """or""",
														        """n""",
														        SPIECE_UNDERLINE + """in""",
														        SPIECE_UNDERLINE + """""",
														        """9""",
														        """2""",
														        """0""",
														        """0""",
														        """0""",
														        """,""",
														        SPIECE_UNDERLINE + """and""",
														        SPIECE_UNDERLINE + """this""",
														        SPIECE_UNDERLINE + """is""",
														        SPIECE_UNDERLINE + """f""",
														        """al""",
														        """s""",
														        """é""",
														        """.""",
														    ]	,			)
														UpperCamelCase__					  =      tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)
														self.assertListEqual(
														    __lowerCAmelCase	,			[
														        value + tokenizer.fairseq_offset
														        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
														        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
														    ]	,			)

														UpperCamelCase__					  =      tokenizer.convert_ids_to_tokens(__lowerCAmelCase	)
														self.assertListEqual(
														    __lowerCAmelCase	,			[
														        SPIECE_UNDERLINE + """I""",
														        SPIECE_UNDERLINE + """was""",
														        SPIECE_UNDERLINE + """b""",
														        """or""",
														        """n""",
														        SPIECE_UNDERLINE + """in""",
														        SPIECE_UNDERLINE + """""",
														        """<unk>""",
														        """2""",
														        """0""",
														        """0""",
														        """0""",
														        """,""",
														        SPIECE_UNDERLINE + """and""",
														        SPIECE_UNDERLINE + """this""",
														        SPIECE_UNDERLINE + """is""",
														        SPIECE_UNDERLINE + """f""",
														        """al""",
														        """s""",
														        """<unk>""",
														        """.""",
														    ]	,			)







							def 					_lowerCamelCase		(    self	):
														if not self.test_slow_tokenizer:
																					# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																					return

														UpperCamelCase__					  =      (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart""", {})
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																												UpperCamelCase__					  =      self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase	,			**__lowerCAmelCase	)
																												UpperCamelCase__					  =      self.tokenizer_class.from_pretrained(__lowerCAmelCase	,			**__lowerCAmelCase	)

																												UpperCamelCase__					  =      tempfile.mkdtemp()

																												UpperCamelCase__					  =      tokenizer_r.save_pretrained(__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.save_pretrained(__lowerCAmelCase	)

																												# Checks it save with the same files + the tokenizer.json file for the fast one
																												self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files	)	)
																												UpperCamelCase__					  =      tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f	)
																												self.assertSequenceEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

																												# Checks everything loads correctly in the same way
																												UpperCamelCase__					  =      tokenizer_r.from_pretrained(__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.from_pretrained(__lowerCAmelCase	)

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(__lowerCAmelCase	,			__lowerCAmelCase	)	)
																																			# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																																			# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																												shutil.rmtree(__lowerCAmelCase	)

																												# Save tokenizer rust, legacy_format=True
																												UpperCamelCase__					  =      tempfile.mkdtemp()

																												UpperCamelCase__					  =      tokenizer_r.save_pretrained(__lowerCAmelCase	,			legacy_format=__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.save_pretrained(__lowerCAmelCase	)

																												# Checks it save with the same files
																												self.assertSequenceEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

																												# Checks everything loads correctly in the same way
																												UpperCamelCase__					  =      tokenizer_r.from_pretrained(__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.from_pretrained(__lowerCAmelCase	)

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(__lowerCAmelCase	,			__lowerCAmelCase	)	)

																												shutil.rmtree(__lowerCAmelCase	)

																												# Save tokenizer rust, legacy_format=False
																												UpperCamelCase__					  =      tempfile.mkdtemp()

																												UpperCamelCase__					  =      tokenizer_r.save_pretrained(__lowerCAmelCase	,			legacy_format=__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.save_pretrained(__lowerCAmelCase	)

																												# Checks it saved the tokenizer.json file
																												self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files	)	)

																												# Checks everything loads correctly in the same way
																												UpperCamelCase__					  =      tokenizer_r.from_pretrained(__lowerCAmelCase	)
																												UpperCamelCase__					  =      tokenizer_p.from_pretrained(__lowerCAmelCase	)

																												# Check special tokens are set accordingly on Rust and Python
																												for key in tokenizer_pp.special_tokens_map:
																																			self.assertTrue(hasattr(__lowerCAmelCase	,			__lowerCAmelCase	)	)

																												shutil.rmtree(__lowerCAmelCase	)






@require_torch
@require_sentencepiece
@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (  unittest.TestCase							):
							snake_case  :   List[Any]      =							"""facebook/mbart-large-en-ro"""
							snake_case  :   Optional[int]      =							[
							    """ UN Chief Says There Is No Military Solution in Syria""",
							    """ Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
							]
							snake_case  :   Optional[Any]      =							[
							    """Şeful ONU declară că nu există o soluţie militară în Siria""",
							    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
							    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
							    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
							]
							snake_case  :   Tuple      =							[82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2, EN_CODE]






							@classmethod
							def 					_lowerCamelCase		(    cls	):
														UpperCamelCase__					  =      MBartTokenizer.from_pretrained(
														    cls.checkpoint_name	,			src_lang="""en_XX"""	,			tgt_lang="""ro_RO"""	)
														UpperCamelCase__					  =      1
														return cls






							def 					_lowerCamelCase		(    self	):
														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]	,			250001	)
														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]	,			250004	)
														self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]	,			250020	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
														self.assertListEqual(self.expected_src_tokens	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														self.assertIn(__lowerCAmelCase	,			self.tokenizer.all_special_ids	)
														UpperCamelCase__					  =      [RO_CODE, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2]
														UpperCamelCase__					  =      self.tokenizer.decode(__lowerCAmelCase	,			skip_special_tokens=__lowerCAmelCase	)
														UpperCamelCase__					  =      self.tokenizer.decode(generated_ids[1:]	,			skip_special_tokens=__lowerCAmelCase	)
														self.assertEqual(__lowerCAmelCase	,			__lowerCAmelCase	)
														self.assertNotIn(self.tokenizer.eos_token	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      ["""this is gunna be a long sentence """ * 20]
														assert isinstance(src_text[0]	,			__lowerCAmelCase	)
														UpperCamelCase__					  =      10
														UpperCamelCase__					  =      self.tokenizer(__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			truncation=__lowerCAmelCase	).input_ids[0]
														self.assertEqual(ids[-2]	,			2	)
														self.assertEqual(ids[-1]	,			__lowerCAmelCase	)
														self.assertEqual(len(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]	)	,			[250026, 250001]	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      tempfile.mkdtemp()
														UpperCamelCase__					  =      self.tokenizer.fairseq_tokens_to_ids
														self.tokenizer.save_pretrained(__lowerCAmelCase	)
														UpperCamelCase__					  =      MBartTokenizer.from_pretrained(__lowerCAmelCase	)
														self.assertDictEqual(new_tok.fairseq_tokens_to_ids	,			__lowerCAmelCase	)






							@require_torch
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      self.tokenizer(self.src_text	,			text_target=self.tgt_text	,			padding=__lowerCAmelCase	,			return_tensors="""pt"""	)
														UpperCamelCase__					  =      shift_tokens_right(batch["""labels"""]	,			self.tokenizer.pad_token_id	)

														# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
														assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
														assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
														assert batch.decoder_input_ids[1][-1] == 2
														assert batch.labels[1][-2:].tolist() == [2, RO_CODE]






							@require_torch
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      self.tokenizer(
														    self.src_text	,			text_target=self.tgt_text	,			padding=__lowerCAmelCase	,			truncation=__lowerCAmelCase	,			max_length=len(self.expected_src_tokens	)	,			return_tensors="""pt"""	,			)

														UpperCamelCase__					  =      shift_tokens_right(batch["""labels"""]	,			self.tokenizer.pad_token_id	)

														self.assertIsInstance(__lowerCAmelCase	,			__lowerCAmelCase	)

														self.assertEqual((2, 14)	,			batch.input_ids.shape	)
														self.assertEqual((2, 14)	,			batch.attention_mask.shape	)
														UpperCamelCase__					  =      batch.input_ids.tolist()[0]
														self.assertListEqual(self.expected_src_tokens	,			__lowerCAmelCase	)
														self.assertEqual(2	,			batch.decoder_input_ids[0, -1]	)  # EOS
														# Test that special tokens are reset
														self.assertEqual(self.tokenizer.prefix_tokens	,			[]	)
														self.assertEqual(self.tokenizer.suffix_tokens	,			[self.tokenizer.eos_token_id, EN_CODE]	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      self.tokenizer(self.src_text	,			padding=__lowerCAmelCase	,			truncation=__lowerCAmelCase	,			max_length=3	,			return_tensors="""pt"""	)
														UpperCamelCase__					  =      self.tokenizer(
														    text_target=self.tgt_text	,			padding=__lowerCAmelCase	,			truncation=__lowerCAmelCase	,			max_length=10	,			return_tensors="""pt"""	)
														UpperCamelCase__					  =      targets["""input_ids"""]
														UpperCamelCase__					  =      shift_tokens_right(__lowerCAmelCase	,			self.tokenizer.pad_token_id	)

														self.assertEqual(batch.input_ids.shape[1]	,			3	)
														self.assertEqual(batch.decoder_input_ids.shape[1]	,			10	)







							@require_torch
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      self.tokenizer._build_translation_inputs(
														    """A test"""	,			return_tensors="""pt"""	,			src_lang="""en_XX"""	,			tgt_lang="""ar_AR"""	)

														self.assertEqual(
														    nested_simplify(__lowerCAmelCase	)	,			{
														        # A, test, EOS, en_XX
														        """input_ids""": [[62, 3034, 2, 250004]],
														        """attention_mask""": [[1, 1, 1, 1]],
														        # ar_AR
														        """forced_bos_token_id""": 250001,
														    }	,			)






import numpy as np
import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModelWithProjection, PreTrainedModel

from ...utils import logging


__A							=logging.get_logger(__name__)




class        _SCREAMING_SNAKE_CASE	( snake_case_		):
	lowerCAmelCase__   =					CLIPConfig

	lowerCAmelCase__   =					['CLIPEncoderLayer']



	def __init__(       self		,    lowercase					)							->			int:
					super().__init__(lowercase					)

					lowerCamelCase_         =     CLIPVisionModelWithProjection(config.vision_config					)

					lowerCamelCase_         =     nn.Linear(config.vision_config.projection_dim		,    1					)
					lowerCamelCase_         =     nn.Linear(config.vision_config.projection_dim		,    1					)



	@torch.no_grad()
	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase		,    lowercase		,    lowercase=0.5		,    lowercase=0.5					)							->			int:
					lowerCamelCase_         =     self.vision_model(lowercase					)[0]

					lowerCamelCase_         =     self.p_head(lowercase					)
					lowerCamelCase_         =     nsfw_detected.flatten()
					lowerCamelCase_         =     nsfw_detected > p_threshold
					lowerCamelCase_         =     nsfw_detected.tolist()

					if any(lowercase					):
									logger.warning(
									    "Potential NSFW content was detected in one or more images. A black image will be returned instead."
									    " Try again with a different prompt and/or seed."					)

					for idx, nsfw_detected_ in enumerate(lowercase					):
									if nsfw_detected_:
													lowerCamelCase_         =     np.zeros(images[idx].shape					)

					lowerCamelCase_         =     self.w_head(lowercase					)
					lowerCamelCase_         =     watermark_detected.flatten()
					lowerCamelCase_         =     watermark_detected > w_threshold
					lowerCamelCase_         =     watermark_detected.tolist()

					if any(lowercase					):
									logger.warning(
									    "Potential watermarked content was detected in one or more images. A black image will be returned instead."
									    " Try again with a different prompt and/or seed."					)

					for idx, watermark_detected_ in enumerate(lowercase					):
									if watermark_detected_:
													lowerCamelCase_         =     np.zeros(images[idx].shape					)

					return images, nsfw_detected, watermark_detected


