



"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase			:			Union[str, Any]										=						get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
		from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_lowercase			:			Tuple										=						2_5_6_0_4_7
_lowercase			:			Union[str, Any]										=						2_5_6_1_4_5

@require_sentencepiece
@require_tokenizers
class 	__SCREAMING_SNAKE_CASE     (				lowerCAmelCase_		,       unittest.TestCase       ):


					'''simple docstring'''





					_a             =							NllbTokenizer
					_a             =							NllbTokenizerFast
					_a             =							True
					_a             =							True
					_a             =							{}



					def 		snake_case	(	self							:  Union[str, Any]					)-> int:
								super().setUp()

								# We have a SentencePiece fixture for testing
								lowerCamelCase__		:		Any		     =NllbTokenizer(lowerCamelCase,  keep_accents=lowerCamelCase					)
								tokenizer.save_pretrained(self.tmpdirname					)



					def 		snake_case	(	self							:  Tuple					)-> List[Any]:
								lowerCamelCase__		:		Union[str, Any]		     =NllbTokenizer(lowerCamelCase,  keep_accents=lowerCamelCase					)

								lowerCamelCase__		:		Any		     =tokenizer.tokenize('''This is a test'''					)
								self.assertListEqual(lowerCamelCase,  ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']					)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(lowerCamelCase					),  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]],  )

								lowerCamelCase__		:		Union[str, Any]		     =tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
								self.assertListEqual(
								    lowerCamelCase,  [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ],  )
								lowerCamelCase__		:		Union[str, Any]		     =tokenizer.convert_tokens_to_ids(lowerCamelCase					)
								self.assertListEqual(
								    lowerCamelCase,  [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ],  )

								lowerCamelCase__		:		str		     =tokenizer.convert_ids_to_tokens(lowerCamelCase					)
								self.assertListEqual(
								    lowerCamelCase,  [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''<unk>''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''<unk>''',
								        '''.''',
								    ],  )



					def 		snake_case	(	self							:  Tuple					)-> Union[str, Any]:
								lowerCamelCase__		:		Optional[int]		     =(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
														lowerCamelCase__		:		List[str]		     =self.rust_tokenizer_class.from_pretrained(lowerCamelCase,  **lowerCamelCase					)
														lowerCamelCase__		:		Any		     =self.tokenizer_class.from_pretrained(lowerCamelCase,  **lowerCamelCase					)

														lowerCamelCase__		:		Any		     =tempfile.mkdtemp()

														lowerCamelCase__		:		Dict		     =tokenizer_r.save_pretrained(lowerCamelCase					)
														lowerCamelCase__		:		str		     =tokenizer_p.save_pretrained(lowerCamelCase					)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files					)					)
														lowerCamelCase__		:		List[str]		     =tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f					)
														self.assertSequenceEqual(lowerCamelCase,  lowerCamelCase					)

														# Checks everything loads correctly in the same way
														lowerCamelCase__		:		Tuple		     =tokenizer_r.from_pretrained(lowerCamelCase					)
														lowerCamelCase__		:		Optional[int]		     =tokenizer_p.from_pretrained(lowerCamelCase					)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowerCamelCase,  lowerCamelCase					)					)

														shutil.rmtree(lowerCamelCase					)

														# Save tokenizer rust, legacy_format=True
														lowerCamelCase__		:		Optional[int]		     =tempfile.mkdtemp()

														lowerCamelCase__		:		Optional[int]		     =tokenizer_r.save_pretrained(lowerCamelCase,  legacy_format=lowerCamelCase					)
														lowerCamelCase__		:		Optional[Any]		     =tokenizer_p.save_pretrained(lowerCamelCase					)

														# Checks it save with the same files
														self.assertSequenceEqual(lowerCamelCase,  lowerCamelCase					)

														# Checks everything loads correctly in the same way
														lowerCamelCase__		:		Dict		     =tokenizer_r.from_pretrained(lowerCamelCase					)
														lowerCamelCase__		:		str		     =tokenizer_p.from_pretrained(lowerCamelCase					)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowerCamelCase,  lowerCamelCase					)					)

														shutil.rmtree(lowerCamelCase					)

														# Save tokenizer rust, legacy_format=False
														lowerCamelCase__		:		Optional[int]		     =tempfile.mkdtemp()

														lowerCamelCase__		:		Optional[Any]		     =tokenizer_r.save_pretrained(lowerCamelCase,  legacy_format=lowerCamelCase					)
														lowerCamelCase__		:		Union[str, Any]		     =tokenizer_p.save_pretrained(lowerCamelCase					)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files					)					)

														# Checks everything loads correctly in the same way
														lowerCamelCase__		:		Any		     =tokenizer_r.from_pretrained(lowerCamelCase					)
														lowerCamelCase__		:		Union[str, Any]		     =tokenizer_p.from_pretrained(lowerCamelCase					)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(lowerCamelCase,  lowerCamelCase					)					)

														shutil.rmtree(lowerCamelCase					)



					@require_torch
					def 		snake_case	(	self							:  Tuple					)-> int:
								if not self.test_seqaseq:
											return

								lowerCamelCase__		:		Tuple		     =self.get_tokenizers()
								for tokenizer in tokenizers:
											with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
														# Longer text that will definitely require truncation.
														lowerCamelCase__		:		Optional[int]		     =[
														    ''' UN Chief Says There Is No Military Solution in Syria''',
														    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
														    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
														    ''' will only worsen the violence and misery for millions of people.''',
														]
														lowerCamelCase__		:		Optional[Any]		     =[
														    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
														    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
														    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
														    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
														]
														try:
																	lowerCamelCase__		:		Dict		     =tokenizer.prepare_seqaseq_batch(
																	    src_texts=lowerCamelCase,  tgt_texts=lowerCamelCase,  max_length=3,  max_target_length=10,  return_tensors='''pt''',  src_lang='''eng_Latn''',  tgt_lang='''ron_Latn''',  )
														except NotImplementedError:
																	return
														self.assertEqual(batch.input_ids.shape[1],  3					)
														self.assertEqual(batch.labels.shape[1],  10					)
														# max_target_length will default to max_length if not specified
														lowerCamelCase__		:		Union[str, Any]		     =tokenizer.prepare_seqaseq_batch(
														    lowerCamelCase,  tgt_texts=lowerCamelCase,  max_length=3,  return_tensors='''pt'''					)
														self.assertEqual(batch.input_ids.shape[1],  3					)
														self.assertEqual(batch.labels.shape[1],  3					)

														lowerCamelCase__		:		Any		     =tokenizer.prepare_seqaseq_batch(
														    src_texts=lowerCamelCase,  max_length=3,  max_target_length=10,  return_tensors='''pt'''					)
														self.assertEqual(batch_encoder_only.input_ids.shape[1],  3					)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1],  3					)
														self.assertNotIn('''decoder_input_ids''',  lowerCamelCase					)



					@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''					)
					def 		snake_case	(	self							:  Tuple					)-> Union[str, Any]:
								pass




					def 		snake_case	(	self							:  Any					)-> int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
														lowerCamelCase__		:		List[str]		     =[AddedToken('''<special>''',  lstrip=lowerCamelCase					)]

														lowerCamelCase__		:		int		     =self.rust_tokenizer_class.from_pretrained(
														    lowerCamelCase,  additional_special_tokens=lowerCamelCase,  **lowerCamelCase					)
														lowerCamelCase__		:		List[Any]		     =tokenizer_r.encode('''Hey this is a <special> token'''					)

														lowerCamelCase__		:		str		     =tokenizer_r.encode('''<special>''',  add_special_tokens=lowerCamelCase					)[0]

														self.assertTrue(special_token_id in r_output					)

														if self.test_slow_tokenizer:
																	lowerCamelCase__		:		Tuple		     =self.rust_tokenizer_class.from_pretrained(
																	    lowerCamelCase,  additional_special_tokens=lowerCamelCase,  **lowerCamelCase,  )
																	lowerCamelCase__		:		Optional[Any]		     =self.tokenizer_class.from_pretrained(
																	    lowerCamelCase,  additional_special_tokens=lowerCamelCase,  **lowerCamelCase					)

																	lowerCamelCase__		:		str		     =tokenizer_p.encode('''Hey this is a <special> token'''					)

																	lowerCamelCase__		:		List[Any]		     =tokenizer_cr.encode('''Hey this is a <special> token'''					)

																	self.assertEqual(lowerCamelCase,  lowerCamelCase					)
																	self.assertEqual(lowerCamelCase,  lowerCamelCase					)
																	self.assertTrue(special_token_id in p_output					)
																	self.assertTrue(special_token_id in cr_output					)





@require_torch
@require_sentencepiece
@require_tokenizers
class 	__SCREAMING_SNAKE_CASE     (				unittest.TestCase       ):


					'''simple docstring'''





					_a             =							'facebook/nllb-200-distilled-600M'
					_a             =							[
					    ' UN Chief Says There Is No Military Solution in Syria',
					    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
					]
					_a             =							[
					    'Şeful ONU declară că nu există o soluţie militară în Siria',
					    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
					    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
					    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
					]
					_a             =							[
					    2_5_6_0_4_7,
					    1_6_2_9_7,
					    1_3_4_4_0_8,
					    8_1_6_5,
					    2_4_8_0_6_6,
					    1_4_7_3_4,
					    9_5_0,
					    1_1_3_5,
					    1_0_5_7_2_1,
					    3_5_7_3,
					    8_3,
					    2_7_3_5_2,
					    1_0_8,
					    4_9_4_8_6,
					    2,
					]



					@classmethod
					def 		snake_case	(	cls							:  int					)-> int:
								lowerCamelCase__		:		NllbTokenizer		     =NllbTokenizer.from_pretrained(
								    cls.checkpoint_name,  src_lang='''eng_Latn''',  tgt_lang='''ron_Latn'''					)
								lowerCamelCase__		:		Optional[Any]		     =1
								return cls



					def 		snake_case	(	self							:  Any					)-> Any:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab'''],  25_6001					)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn'''],  25_6002					)
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn'''],  25_6057					)



					def 		snake_case	(	self							:  Any					)-> Any:
								lowerCamelCase__		:		Any		     =self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
								self.assertListEqual(self.expected_src_tokens,  lowerCamelCase					)



					def 		snake_case	(	self							:  Union[str, Any]					)-> Optional[Any]:
								self.assertIn(lowerCamelCase,  self.tokenizer.all_special_ids					)
								# fmt: off
								lowerCamelCase__		:		Tuple		     =[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
								# fmt: on

								lowerCamelCase__		:		Any		     =self.tokenizer.decode(lowerCamelCase,  skip_special_tokens=lowerCamelCase					)
								lowerCamelCase__		:		Dict		     =self.tokenizer.decode(generated_ids[1:],  skip_special_tokens=lowerCamelCase					)
								self.assertEqual(lowerCamelCase,  lowerCamelCase					)
								self.assertNotIn(self.tokenizer.eos_token,  lowerCamelCase					)



					def 		snake_case	(	self							:  Union[str, Any]					)-> str:
								lowerCamelCase__		:		Optional[Any]		     =['''this is gunna be a long sentence ''' * 20]
								assert isinstance(src_text[0],  lowerCamelCase					)
								lowerCamelCase__		:		Dict		     =10
								lowerCamelCase__		:		int		     =self.tokenizer(lowerCamelCase,  max_length=lowerCamelCase,  truncation=lowerCamelCase					).input_ids[0]
								self.assertEqual(ids[-1],  2					)
								self.assertEqual(ids[0],  lowerCamelCase					)
								self.assertEqual(len(lowerCamelCase					),  lowerCamelCase					)



					def 		snake_case	(	self							:  Dict					)-> Dict:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']					),  [25_6203, 3]					)



					def 		snake_case	(	self							:  List[str]					)-> List[str]:
								lowerCamelCase__		:		Union[str, Any]		     =tempfile.mkdtemp()
								lowerCamelCase__		:		int		     =self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(lowerCamelCase					)
								lowerCamelCase__		:		Optional[Any]		     =NllbTokenizer.from_pretrained(lowerCamelCase					)
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids,  lowerCamelCase					)



					@require_torch
					def 		snake_case	(	self							:  int					)-> Dict:
								lowerCamelCase__		:		List[Any]		     =self.tokenizer(
								    self.src_text,  text_target=self.tgt_text,  padding=lowerCamelCase,  truncation=lowerCamelCase,  max_length=len(self.expected_src_tokens					),  return_tensors='''pt''',  )
								lowerCamelCase__		:		Optional[int]		     =shift_tokens_right(
								    batch['''labels'''],  self.tokenizer.pad_token_id,  self.tokenizer.lang_code_to_id['''ron_Latn''']					)

								self.assertIsInstance(lowerCamelCase,  lowerCamelCase					)

								self.assertEqual((2, 15),  batch.input_ids.shape					)
								self.assertEqual((2, 15),  batch.attention_mask.shape					)
								lowerCamelCase__		:		Optional[int]		     =batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens,  lowerCamelCase					)
								self.assertEqual(lowerCamelCase,  batch.decoder_input_ids[0, 0]					)  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens,  [EN_CODE]					)
								self.assertEqual(self.tokenizer.suffix_tokens,  [self.tokenizer.eos_token_id]					)



					def 		snake_case	(	self							:  Tuple					)-> Optional[int]:
								lowerCamelCase__		:		List[Any]		     =self.tokenizer(self.src_text,  padding=lowerCamelCase,  truncation=lowerCamelCase,  max_length=3,  return_tensors='''pt'''					)
								lowerCamelCase__		:		Any		     =self.tokenizer(
								    text_target=self.tgt_text,  padding=lowerCamelCase,  truncation=lowerCamelCase,  max_length=10,  return_tensors='''pt'''					)
								lowerCamelCase__		:		Tuple		     =targets['''input_ids''']
								lowerCamelCase__		:		Dict		     =shift_tokens_right(
								    lowerCamelCase,  self.tokenizer.pad_token_id,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang],  )

								self.assertEqual(batch.input_ids.shape[1],  3					)
								self.assertEqual(batch.decoder_input_ids.shape[1],  10					)



					@require_torch
					def 		snake_case	(	self							:  List[str]					)-> List[Any]:
								lowerCamelCase__		:		int		     =self.tokenizer._build_translation_inputs(
								    '''A test''',  return_tensors='''pt''',  src_lang='''eng_Latn''',  tgt_lang='''fra_Latn'''					)

								self.assertEqual(
								    nested_simplify(lowerCamelCase					),  {
								        # A, test, EOS, en_XX
								        '''input_ids''': [[25_6047, 70, 7356, 2]],
								        '''attention_mask''': [[1, 1, 1, 1]],
								        # ar_AR
								        '''forced_bos_token_id''': 25_6057,
								    },  )




					@require_torch
					def 		snake_case	(	self							:  str					)-> Any:
								lowerCamelCase__		:		Optional[Any]		     =True
								lowerCamelCase__		:		Any		     =self.tokenizer(
								    '''UN Chief says there is no military solution in Syria''',  src_lang='''eng_Latn''',  tgt_lang='''fra_Latn'''					)
								self.assertEqual(
								    inputs.input_ids,  [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]					)

								lowerCamelCase__		:		Optional[int]		     =False
								lowerCamelCase__		:		Optional[Any]		     =self.tokenizer(
								    '''UN Chief says there is no military solution in Syria''',  src_lang='''eng_Latn''',  tgt_lang='''fra_Latn'''					)
								self.assertEqual(
								    inputs.input_ids,  [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]					)









"""simple docstring"""






import numpy as np
import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModelWithProjection, PreTrainedModel

from ...utils import logging


_lowercase			:			int										=						logging.get_logger(__name__)

class 	__SCREAMING_SNAKE_CASE     (				lowerCAmelCase_       ):


					'''simple docstring'''





					_a             =							CLIPConfig

					_a             =							['CLIPEncoderLayer']



					def __init__(	self							:  int,  lowerCamelCase							:  CLIPConfig					)-> List[Any]:
								super().__init__(lowerCamelCase					)

								lowerCamelCase__		:		Dict		     =CLIPVisionModelWithProjection(config.vision_config					)

								lowerCamelCase__		:		Dict		     =nn.Linear(config.vision_config.projection_dim,  1					)
								lowerCamelCase__		:		Union[str, Any]		     =nn.Linear(config.vision_config.projection_dim,  1					)




					@torch.no_grad()
					def 		snake_case	(	self							:  Dict,  lowerCamelCase							:  Optional[Any],  lowerCamelCase							:  List[str],  lowerCamelCase							:  List[Any]=0.5,  lowerCamelCase							:  Optional[Any]=0.5					)-> Optional[int]:
								lowerCamelCase__		:		Dict		     =self.vision_model(lowerCamelCase					)[0]

								lowerCamelCase__		:		List[Any]		     =self.p_head(lowerCamelCase					)
								lowerCamelCase__		:		Optional[Any]		     =nsfw_detected.flatten()
								lowerCamelCase__		:		Any		     =nsfw_detected > p_threshold
								lowerCamelCase__		:		Optional[Any]		     =nsfw_detected.tolist()

								if any(lowerCamelCase					):
											logger.warning(
											    '''Potential NSFW content was detected in one or more images. A black image will be returned instead.'''
											    ''' Try again with a different prompt and/or seed.'''					)

								for idx, nsfw_detected_ in enumerate(lowerCamelCase					):
											if nsfw_detected_:
														lowerCamelCase__		:		Tuple		     =np.zeros(images[idx].shape					)

								lowerCamelCase__		:		str		     =self.w_head(lowerCamelCase					)
								lowerCamelCase__		:		List[str]		     =watermark_detected.flatten()
								lowerCamelCase__		:		Optional[Any]		     =watermark_detected > w_threshold
								lowerCamelCase__		:		int		     =watermark_detected.tolist()

								if any(lowerCamelCase					):
											logger.warning(
											    '''Potential watermarked content was detected in one or more images. A black image will be returned instead.'''
											    ''' Try again with a different prompt and/or seed.'''					)

								for idx, watermark_detected_ in enumerate(lowerCamelCase					):
											if watermark_detected_:
														lowerCamelCase__		:		Optional[int]		     =np.zeros(images[idx].shape					)

								return images, nsfw_detected, watermark_detected




