
"""simple docstring"""



import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__          =	get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
    from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCamelCase__          =	5_00_03
UpperCamelCase__          =	5_00_02





@require_sentencepiece
@require_tokenizers
class 				a			(					lowercase						,		unittest.TestCase      ):
      UpperCamelCase							:   List[str]           =     PLBartTokenizer
      UpperCamelCase							:   Tuple           =     None
      UpperCamelCase							:   int           =     False



      def  __snake_case							(   self      ):
          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase__	:      str     							=					PLBartTokenizer(UpperCamelCase_     ,	language_codes='base'     ,	keep_accents=UpperCamelCase_      )
          tokenizer.save_pretrained(self.tmpdirname      )



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Dict     							=					PLBartTokenizer(UpperCamelCase_     ,	language_codes='base'     ,	keep_accents=UpperCamelCase_      )

          UpperCAmelCase__	:      Tuple     							=					tokenizer.tokenize('This is a test'      )
          self.assertListEqual(UpperCamelCase_     ,	['▁This', '▁is', '▁a', '▁t', 'est']      )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(UpperCamelCase_      )     ,	[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,	)

          UpperCAmelCase__	:      List[Any]     							=					tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ]     ,	)
          UpperCAmelCase__	:      Dict     							=					tokenizer.convert_tokens_to_ids(UpperCamelCase_      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]     ,	)

          UpperCAmelCase__	:      Optional[Any]     							=					tokenizer.convert_ids_to_tokens(UpperCamelCase_      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '<unk>',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '<unk>',
                  '.',
              ]     ,	)

          UpperCAmelCase__	:      Tuple     							=					tokenizer.vocab_size
          UpperCAmelCase__	:      List[str]     							=					[tokenizer.convert_ids_to_tokens(UpperCamelCase_      ) for x in range(end - 4     ,	UpperCamelCase_      )]

          self.assertListEqual(UpperCamelCase_     ,	['__java__', '__python__', '__en_XX__', '<mask>']      )

          UpperCAmelCase__	:      List[Any]     							=					'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
          UpperCAmelCase__	:      Tuple     							=					tokenizer(UpperCamelCase_      ).input_ids
          self.assertEqual(
              tokenizer.decode(UpperCamelCase_     ,	skip_special_tokens=UpperCamelCase_     ,	clean_up_tokenization_spaces=UpperCamelCase_      )     ,	UpperCamelCase_     ,	)



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      List[str]     							=					PLBartTokenizer(UpperCamelCase_     ,	language_codes='multi'     ,	keep_accents=UpperCamelCase_      )

          UpperCAmelCase__	:      List[str]     							=					tokenizer.tokenize('This is a test'      )
          self.assertListEqual(UpperCamelCase_     ,	['▁This', '▁is', '▁a', '▁t', 'est']      )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(UpperCamelCase_      )     ,	[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,	)

          UpperCAmelCase__	:      Dict     							=					tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ]     ,	)
          UpperCAmelCase__	:      Tuple     							=					tokenizer.convert_tokens_to_ids(UpperCamelCase_      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]     ,	)

          UpperCAmelCase__	:      List[str]     							=					tokenizer.convert_ids_to_tokens(UpperCamelCase_      )
          self.assertListEqual(
              UpperCamelCase_     ,	[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '<unk>',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '<unk>',
                  '.',
              ]     ,	)
          UpperCAmelCase__	:      str     							=					tokenizer.vocab_size
          UpperCAmelCase__	:      int     							=					[tokenizer.convert_ids_to_tokens(UpperCamelCase_      ) for x in range(end - 7     ,	UpperCamelCase_      )]

          self.assertListEqual(
              UpperCamelCase_     ,	['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']      )
          UpperCAmelCase__	:      List[Any]     							=					'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
          UpperCAmelCase__	:      Optional[int]     							=					tokenizer(UpperCamelCase_      ).input_ids
          self.assertEqual(
              tokenizer.decode(UpperCamelCase_     ,	skip_special_tokens=UpperCamelCase_     ,	clean_up_tokenization_spaces=UpperCamelCase_      )     ,	UpperCamelCase_     ,	)





@require_torch
@require_sentencepiece
@require_tokenizers
class 				a			(					unittest.TestCase      ):
      UpperCamelCase							:   Any           =     """uclanlp/plbart-python-en_XX"""
      UpperCamelCase							:   Optional[Any]           =     [
          """def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])""",
          """def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""",
      ]
      UpperCamelCase							:   Any           =     [
          """Returns the maximum value of a b c.""",
          """Sums the values of a b c.""",
      ]
      UpperCamelCase							:   Union[str, Any]           =     [
          1_3_4,
          5_4_5_2,
          3_3_4_6_0,
          3_3_4_4_1,
          3_3_4_6_3,
          3_3_4_6_5,
          3_3_4_6_3,
          3_3_4_4_9,
          9_8_8,
          2_0,
          3_3_4_5_6,
          1_9,
          3_3_4_5_6,
          7_7_1,
          3_9,
          4_2_5_8,
          8_8_9,
          3_3_1_8,
          3_3_4_4_1,
          3_3_4_6_3,
          3_3_4_6_5,
          3_3_4_6_3,
          3_3_4_4_9,
          2_4_7_1,
          2,
          PYTHON_CODE,
      ]



      @classmethod
      def  __snake_case							(   cls      ):
          UpperCAmelCase__	:      Union[str, Any]     							=					PLBartTokenizer.from_pretrained(
              cls.checkpoint_name     ,	language_codes='base'     ,	src_lang='python'     ,	tgt_lang='en_XX'      )
          UpperCAmelCase__	:      Optional[int]     							=					1
          return cls



      def  __snake_case							(   self      ):
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']     ,	50_001      )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']     ,	50_002      )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']     ,	50_003      )



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      str     							=					self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
          self.assertListEqual(self.expected_src_tokens     ,	UpperCamelCase_      )



      def  __snake_case							(   self      ):
          self.assertIn(UpperCamelCase_     ,	self.tokenizer.all_special_ids      )
          UpperCAmelCase__	:      int     							=					[EN_CODE, 9_037, 33_442, 57, 752, 153, 14, 56, 18, 9, 2]
          UpperCAmelCase__	:      List[str]     							=					self.tokenizer.decode(UpperCamelCase_     ,	skip_special_tokens=UpperCamelCase_      )
          UpperCAmelCase__	:      Optional[Any]     							=					self.tokenizer.decode(generated_ids[1:]     ,	skip_special_tokens=UpperCamelCase_      )
          self.assertEqual(UpperCamelCase_     ,	UpperCamelCase_      )
          self.assertNotIn(self.tokenizer.eos_token     ,	UpperCamelCase_      )



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Optional[int]     							=					['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
          self.assertIsInstance(src_text[0]     ,	UpperCamelCase_      )
          UpperCAmelCase__	:      Any     							=					10
          UpperCAmelCase__	:      Optional[Any]     							=					self.tokenizer(UpperCamelCase_     ,	max_length=UpperCamelCase_     ,	truncation=UpperCamelCase_      ).input_ids[0]
          self.assertEqual(ids[-2]     ,	2      )
          self.assertEqual(ids[-1]     ,	UpperCamelCase_      )
          self.assertEqual(len(UpperCamelCase_      )     ,	UpperCamelCase_      )



      def  __snake_case							(   self      ):
          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']      )     ,	[50_004, 50_001]      )



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Optional[int]     							=					tempfile.mkdtemp()
          UpperCAmelCase__	:      int     							=					self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(UpperCamelCase_      )
          UpperCAmelCase__	:      List[Any]     							=					PLBartTokenizer.from_pretrained(UpperCamelCase_      )
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,	UpperCamelCase_      )



      @require_torch
      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Any     							=					self.tokenizer(self.src_text     ,	text_target=self.tgt_text     ,	padding=UpperCamelCase_     ,	return_tensors='pt'      )
          UpperCAmelCase__	:      Dict     							=					shift_tokens_right(batch['labels']     ,	self.tokenizer.pad_token_id      )

          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          self.assertEqual(batch.input_ids[1][-2:].tolist()     ,	[2, PYTHON_CODE]      )
          self.assertEqual(batch.decoder_input_ids[1][0]     ,	UpperCamelCase_      )
          self.assertEqual(batch.decoder_input_ids[1][-1]     ,	2      )
          self.assertEqual(batch.labels[1][-2:].tolist()     ,	[2, EN_CODE]      )



      @require_torch
      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Optional[Any]     							=					self.tokenizer(
              self.src_text     ,	text_target=self.tgt_text     ,	padding=UpperCamelCase_     ,	truncation=UpperCamelCase_     ,	max_length=len(self.expected_src_tokens      )     ,	return_tensors='pt'     ,	)
          UpperCAmelCase__	:      Dict     							=					shift_tokens_right(batch['labels']     ,	self.tokenizer.pad_token_id      )

          self.assertIsInstance(UpperCamelCase_     ,	UpperCamelCase_      )

          self.assertEqual((2, 26)     ,	batch.input_ids.shape      )
          self.assertEqual((2, 26)     ,	batch.attention_mask.shape      )
          UpperCAmelCase__	:      Union[str, Any]     							=					batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens     ,	UpperCamelCase_      )
          self.assertEqual(2     ,	batch.decoder_input_ids[0, -1]      )  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens     ,	[]      )
          self.assertEqual(self.tokenizer.suffix_tokens     ,	[self.tokenizer.eos_token_id, PYTHON_CODE]      )



      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Union[str, Any]     							=					self.tokenizer(self.src_text     ,	padding=UpperCamelCase_     ,	truncation=UpperCamelCase_     ,	max_length=3     ,	return_tensors='pt'      )
          UpperCAmelCase__	:      Optional[int]     							=					self.tokenizer(
              text_target=self.tgt_text     ,	padding=UpperCamelCase_     ,	truncation=UpperCamelCase_     ,	max_length=10     ,	return_tensors='pt'      )
          UpperCAmelCase__	:      List[str]     							=					targets['input_ids']
          UpperCAmelCase__	:      List[str]     							=					shift_tokens_right(UpperCamelCase_     ,	self.tokenizer.pad_token_id      )

          self.assertEqual(batch.input_ids.shape[1]     ,	3      )
          self.assertEqual(batch.decoder_input_ids.shape[1]     ,	10      )



      @require_torch
      def  __snake_case							(   self      ):
          UpperCAmelCase__	:      Optional[Any]     							=					self.tokenizer._build_translation_inputs(
              'A test'     ,	return_tensors='pt'     ,	src_lang='en_XX'     ,	tgt_lang='java'      )

          self.assertEqual(
              nested_simplify(UpperCamelCase_      )     ,	{
                  # A, test, EOS, en_XX
                  'input_ids': [[150, 242, 2, 50_003]],
                  'attention_mask': [[1, 1, 1, 1]],
                  # java
                  'forced_bos_token_id': 50_001,
              }     ,	)




import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


lowercase_   							=  logging.get_logger(__name__)
def 	a (     A__      :  Optional[int] ,  A__      :  Union[str, Any]	)       -> int:





      """simple docstring"""



      _lowercase      			=nn.functional.normalize(A__	)
      _lowercase      			=nn.functional.normalize(A__	)
      return torch.mm(A__ ,  normalized_text_embeds.t()	)







class 					__lowerCAmelCase	(   SCREAMING_SNAKE_CASE				):
  _a		     =    CLIPConfig

  _a		     =    ["""CLIPEncoderLayer"""]




  def __init__(							self				,  lowerCAmelCase   )    ->					Union[str, Any]:
        '''simple docstring'''






        super().__init__(lowerCAmelCase   )

        _lowercase      			=CLIPVisionModel(config.vision_config   )
        _lowercase      			=nn.Linear(config.vision_config.hidden_size				,  config.projection_dim				,  bias=lowerCAmelCase   )

        _lowercase      			=nn.Parameter(torch.ones(17				,  config.projection_dim   )				,  requires_grad=lowerCAmelCase   )
        _lowercase      			=nn.Parameter(torch.ones(3				,  config.projection_dim   )				,  requires_grad=lowerCAmelCase   )

        _lowercase      			=nn.Parameter(torch.ones(17   )				,  requires_grad=lowerCAmelCase   )
        _lowercase      			=nn.Parameter(torch.ones(3   )				,  requires_grad=lowerCAmelCase   )




  @torch.no_grad()
  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase   )    ->					Optional[int]:
        '''simple docstring'''






        _lowercase      			=self.vision_model(lowerCAmelCase   )[1]  # pooled_output
        _lowercase      			=self.visual_projection(lowerCAmelCase   )

        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
        _lowercase      			=cosine_distance(lowerCAmelCase				,  self.special_care_embeds   ).cpu().float().numpy()
        _lowercase      			=cosine_distance(lowerCAmelCase				,  self.concept_embeds   ).cpu().float().numpy()

        _lowercase      			=[]
        _lowercase      			=image_embeds.shape[0]
        for i in range(lowerCAmelCase   ):
              _lowercase      			={'special_scores': {}, 'special_care': [], 'concept_scores': {}, 'bad_concepts': []}

              # increase this value to create a stronger `nfsw` filter
              # at the cost of increasing the possibility of filtering benign images
              _lowercase      			=0.0

              for concept_idx in range(len(special_cos_dist[0]   )   ):
                    _lowercase      			=special_cos_dist[i][concept_idx]
                    _lowercase      			=self.special_care_embeds_weights[concept_idx].item()
                    _lowercase      			=round(concept_cos - concept_threshold + adjustment				,  3   )
                    if result_img["special_scores"][concept_idx] > 0:
                          result_img["special_care"].append({concept_idx, result_img['special_scores'][concept_idx]}   )
                          _lowercase      			=0.01

              for concept_idx in range(len(cos_dist[0]   )   ):
                    _lowercase      			=cos_dist[i][concept_idx]
                    _lowercase      			=self.concept_embeds_weights[concept_idx].item()
                    _lowercase      			=round(concept_cos - concept_threshold + adjustment				,  3   )
                    if result_img["concept_scores"][concept_idx] > 0:
                          result_img["bad_concepts"].append(lowerCAmelCase   )

              result.append(lowerCAmelCase   )

        _lowercase      			=[len(res['bad_concepts']   ) > 0 for res in result]

        return images, has_nsfw_concepts




  @torch.no_grad()
  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase   )    ->					Tuple:
        '''simple docstring'''






        _lowercase      			=self.vision_model(lowerCAmelCase   )[1]  # pooled_output
        _lowercase      			=self.visual_projection(lowerCAmelCase   )

        _lowercase      			=cosine_distance(lowerCAmelCase				,  self.special_care_embeds   )
        _lowercase      			=cosine_distance(lowerCAmelCase				,  self.concept_embeds   )

        # increase this value to create a stronger `nsfw` filter
        # at the cost of increasing the possibility of filtering benign images
        _lowercase      			=0.0

        _lowercase      			=special_cos_dist - self.special_care_embeds_weights + adjustment
        # special_scores = special_scores.round(decimals=3)
        _lowercase      			=torch.any(special_scores > 0				,  dim=1   )
        _lowercase      			=special_care * 0.01
        _lowercase      			=special_adjustment.unsqueeze(1   ).expand(-1				,  cos_dist.shape[1]   )

        _lowercase      			=(cos_dist - self.concept_embeds_weights) + special_adjustment
        # concept_scores = concept_scores.round(decimals=3)
        _lowercase      			=torch.any(concept_scores > 0				,  dim=1   )

        return images, has_nsfw_concepts


