


'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import CLIPSegProcessor, ViTImageProcessor



@require_vision
class 			a       (	unittest.TestCase ):


     """simple docstring"""






     def 				lowerCamelCase__				(       self  :	Tuple    )		->							Dict:
          __UpperCAmelCase			:				Optional[Any]													=    tempfile.mkdtemp()

          # fmt: off
          __UpperCAmelCase			:				Dict													=    ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
          # fmt: on
          __UpperCAmelCase			:				Optional[int]													=    dict(zip(snake_case							,				range(len(snake_case    )    )    )    )
          __UpperCAmelCase			:				List[str]													=    ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
          __UpperCAmelCase			:				Optional[Any]													=    {'''unk_token''': '''<unk>'''}

          __UpperCAmelCase			:				Optional[int]													=    os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES['''vocab_file''']    )
          __UpperCAmelCase			:				Tuple													=    os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES['''merges_file''']    )
          with open(self.vocab_file							,				'''w'''							,				encoding='''utf-8'''    ) as fp:
               fp.write(json.dumps(snake_case    ) + '''\n'''    )
          with open(self.merges_file							,				'''w'''							,				encoding='''utf-8'''    ) as fp:
               fp.write('''\n'''.join(snake_case    )    )

          __UpperCAmelCase			:				Optional[Any]													=    {
              '''do_resize''': True,
              '''size''': 20,
              '''do_center_crop''': True,
              '''crop_size''': 18,
              '''do_normalize''': True,
              '''image_mean''': [0.48_145_466, 0.4_578_275, 0.40_821_073],
              '''image_std''': [0.26_862_954, 0.26_130_258, 0.27_577_711],
          }
          __UpperCAmelCase			:				Tuple													=    os.path.join(self.tmpdirname							,				snake_case    )
          with open(self.image_processor_file							,				'''w'''							,				encoding='''utf-8'''    ) as fp:
               json.dump(snake_case							,				snake_case    )

     def 				lowerCamelCase__				(       self  :	Union[str, Any]							,				**snake_case  :	str    )		->							List[str]:
          return CLIPTokenizer.from_pretrained(self.tmpdirname							,				**snake_case    )

     def 				lowerCamelCase__				(       self  :	Dict							,				**snake_case  :	Any    )		->							Dict:
          return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,				**snake_case    )

     def 				lowerCamelCase__				(       self  :	str							,				**snake_case  :	Optional[int]    )		->							Optional[int]:
          return ViTImageProcessor.from_pretrained(self.tmpdirname							,				**snake_case    )

     def 				lowerCamelCase__				(       self  :	List[str]    )		->							List[Any]:
          shutil.rmtree(self.tmpdirname    )

     def 				lowerCamelCase__				(       self  :	Any    )		->							Union[str, Any]:
          __UpperCAmelCase			:				int													=    [np.random.randint(255							,				size=(3, 30, 400)							,				dtype=np.uinta    )]

          __UpperCAmelCase			:				str													=    [Image.fromarray(np.moveaxis(snake_case							,				0							,				-1    )    ) for x in image_inputs]

          return image_inputs

     def 				lowerCamelCase__				(       self  :	List[Any]    )		->							List[str]:
          __UpperCAmelCase			:				List[Any]													=    self.get_tokenizer()
          __UpperCAmelCase			:				Any													=    self.get_rust_tokenizer()
          __UpperCAmelCase			:				Union[str, Any]													=    self.get_image_processor()

          __UpperCAmelCase			:				Optional[int]													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )
          processor_slow.save_pretrained(self.tmpdirname    )
          __UpperCAmelCase			:				str													=    CLIPSegProcessor.from_pretrained(self.tmpdirname							,				use_fast=snake_case    )

          __UpperCAmelCase			:				Dict													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )
          processor_fast.save_pretrained(self.tmpdirname    )
          __UpperCAmelCase			:				Optional[int]													=    CLIPSegProcessor.from_pretrained(self.tmpdirname    )

          self.assertEqual(processor_slow.tokenizer.get_vocab()							,				tokenizer_slow.get_vocab()    )
          self.assertEqual(processor_fast.tokenizer.get_vocab()							,				tokenizer_fast.get_vocab()    )
          self.assertEqual(tokenizer_slow.get_vocab()							,				tokenizer_fast.get_vocab()    )
          self.assertIsInstance(processor_slow.tokenizer							,				snake_case    )
          self.assertIsInstance(processor_fast.tokenizer							,				snake_case    )

          self.assertEqual(processor_slow.image_processor.to_json_string()							,				image_processor.to_json_string()    )
          self.assertEqual(processor_fast.image_processor.to_json_string()							,				image_processor.to_json_string()    )
          self.assertIsInstance(processor_slow.image_processor							,				snake_case    )
          self.assertIsInstance(processor_fast.image_processor							,				snake_case    )

     def 				lowerCamelCase__				(       self  :	int    )		->							str:
          __UpperCAmelCase			:				Dict													=    CLIPSegProcessor(tokenizer=self.get_tokenizer()							,				image_processor=self.get_image_processor()    )
          processor.save_pretrained(self.tmpdirname    )

          __UpperCAmelCase			:				List[str]													=    self.get_tokenizer(bos_token='''(BOS)'''							,				eos_token='''(EOS)'''    )
          __UpperCAmelCase			:				List[Any]													=    self.get_image_processor(do_normalize=snake_case							,				padding_value=1.0    )

          __UpperCAmelCase			:				Optional[int]													=    CLIPSegProcessor.from_pretrained(
              self.tmpdirname							,				bos_token='''(BOS)'''							,				eos_token='''(EOS)'''							,				do_normalize=snake_case							,				padding_value=1.0    )

          self.assertEqual(processor.tokenizer.get_vocab()							,				tokenizer_add_kwargs.get_vocab()    )
          self.assertIsInstance(processor.tokenizer							,				snake_case    )

          self.assertEqual(processor.image_processor.to_json_string()							,				image_processor_add_kwargs.to_json_string()    )
          self.assertIsInstance(processor.image_processor							,				snake_case    )

     def 				lowerCamelCase__				(       self  :	Optional[Any]    )		->							Union[str, Any]:
          __UpperCAmelCase			:				Any													=    self.get_image_processor()
          __UpperCAmelCase			:				str													=    self.get_tokenizer()

          __UpperCAmelCase			:				int													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )

          __UpperCAmelCase			:				Optional[int]													=    self.prepare_image_inputs()

          __UpperCAmelCase			:				int													=    image_processor(snake_case							,				return_tensors='''np'''    )
          __UpperCAmelCase			:				Union[str, Any]													=    processor(images=snake_case							,				return_tensors='''np'''    )

          for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum()							,				input_processor[key].sum()							,				delta=1E-2    )

     def 				lowerCamelCase__				(       self  :	Optional[int]    )		->							int:
          __UpperCAmelCase			:				int													=    self.get_image_processor()
          __UpperCAmelCase			:				str													=    self.get_tokenizer()

          __UpperCAmelCase			:				Tuple													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )

          __UpperCAmelCase			:				int													=    '''lower newer'''

          __UpperCAmelCase			:				Tuple													=    processor(text=snake_case    )

          __UpperCAmelCase			:				str													=    tokenizer(snake_case    )

          for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]							,				encoded_processor[key]    )

     def 				lowerCamelCase__				(       self  :	Union[str, Any]    )		->							Dict:
          __UpperCAmelCase			:				Optional[int]													=    self.get_image_processor()
          __UpperCAmelCase			:				List[str]													=    self.get_tokenizer()

          __UpperCAmelCase			:				Union[str, Any]													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )

          __UpperCAmelCase			:				Optional[int]													=    '''lower newer'''
          __UpperCAmelCase			:				Tuple													=    self.prepare_image_inputs()

          __UpperCAmelCase			:				Dict													=    processor(text=snake_case							,				images=snake_case    )

          self.assertListEqual(list(inputs.keys()    )							,				['''input_ids''', '''attention_mask''', '''pixel_values''']    )

          # test if it raises when no input is passed
          with pytest.raises(snake_case    ):
               processor()

     def 				lowerCamelCase__				(       self  :	Any    )		->							Optional[Any]:
          __UpperCAmelCase			:				Optional[int]													=    self.get_image_processor()
          __UpperCAmelCase			:				List[str]													=    self.get_tokenizer()

          __UpperCAmelCase			:				Any													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )

          __UpperCAmelCase			:				Tuple													=    self.prepare_image_inputs()
          __UpperCAmelCase			:				Any													=    self.prepare_image_inputs()

          __UpperCAmelCase			:				Optional[Any]													=    processor(images=snake_case							,				visual_prompt=snake_case    )

          self.assertListEqual(list(inputs.keys()    )							,				['''pixel_values''', '''conditional_pixel_values''']    )

          # test if it raises when no input is passed
          with pytest.raises(snake_case    ):
               processor()






     def 				lowerCamelCase__				(       self  :	int    )		->							Optional[int]:
          __UpperCAmelCase			:				Any													=    self.get_image_processor()
          __UpperCAmelCase			:				Dict													=    self.get_tokenizer()

          __UpperCAmelCase			:				Tuple													=    CLIPSegProcessor(tokenizer=snake_case							,				image_processor=snake_case    )

          __UpperCAmelCase			:				List[str]													=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          __UpperCAmelCase			:				List[str]													=    processor.batch_decode(snake_case    )
          __UpperCAmelCase			:				str													=    tokenizer.batch_decode(snake_case    )

          self.assertListEqual(snake_case							,				snake_case    )



'''simple docstring'''



import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
      import tensorflow as tf

if is_tensorflow_text_available():
      from transformers.models.bert import TFBertTokenizer


__UpperCAmelCase    :Optional[int]				       =     ["bert-base-uncased", "bert-base-cased"]
__UpperCAmelCase    :str				       =     "hf-internal-testing/tiny-bert-tf-only"

if is_tf_available():



      class 			a       (	tf.keras.Model ):


           """simple docstring"""






           def __init__(       self  :	List[str]							,				snake_case  :	List[str]    )		->							str:
                super().__init__()
                __UpperCAmelCase			:				List[str]													=    tokenizer
                __UpperCAmelCase			:				List[Any]													=    AutoConfig.from_pretrained(snake_case    )
                __UpperCAmelCase			:				int													=    TFAutoModel.from_config(snake_case    )






           def 				lowerCamelCase__				(       self  :	List[Any]							,				snake_case  :	Optional[int]    )		->							Optional[Any]:
                __UpperCAmelCase			:				List[Any]													=    self.tokenizer(snake_case    )
                __UpperCAmelCase			:				Optional[Any]													=    self.bert(**snake_case    )
                return out["pooler_output"]



@require_tf
@require_tensorflow_text
class 			a       (	unittest.TestCase ):


     """simple docstring"""






     def 				lowerCamelCase__				(       self  :	Optional[int]    )		->							List[str]:
          super().setUp()

          __UpperCAmelCase			:				Tuple													=    [
              BertTokenizer.from_pretrained(snake_case    ) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
          ]  # repeat for when fast_bert_tokenizer=false
          __UpperCAmelCase			:				Any													=    [TFBertTokenizer.from_pretrained(snake_case    ) for checkpoint in TOKENIZER_CHECKPOINTS] + [
              TFBertTokenizer.from_pretrained(snake_case							,				use_fast_bert_tokenizer=snake_case    )
              for checkpoint in TOKENIZER_CHECKPOINTS
          ]
          assert len(self.tokenizers    ) == len(self.tf_tokenizers    )

          __UpperCAmelCase			:				Optional[int]													=    [
              '''This is a straightforward English test sentence.''',
              '''This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.''',
              '''Now we\'re going to add some Chinese: 一 二 三 一二三''',
              '''And some much more rare Chinese: 齉 堃 齉堃''',
              '''Je vais aussi écrire en français pour tester les accents''',
              '''Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ''',
          ]
          __UpperCAmelCase			:				Optional[int]													=    list(zip(self.test_sentences							,				self.test_sentences[::-1]    )    )

     def 				lowerCamelCase__				(       self  :	Optional[int]    )		->							Optional[Any]:
          for tokenizer, tf_tokenizer in zip(self.tokenizers							,				self.tf_tokenizers    ):
               for test_inputs in (self.test_sentences, self.paired_sentences):
                    __UpperCAmelCase			:				Any													=    tokenizer(snake_case							,				return_tensors='''tf'''							,				padding='''longest'''    )
                    __UpperCAmelCase			:				Optional[int]													=    tf_tokenizer(snake_case    )

                    for key in python_outputs.keys():
                         self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape    )    )
                         self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]							,				tf.intaa    ) == tf_outputs[key]    )    )

     @slow
     def 				lowerCamelCase__				(       self  :	List[Any]    )		->							str:
          for tf_tokenizer in self.tf_tokenizers:
               __UpperCAmelCase			:				Any													=    tf_tokenizer(self.paired_sentences    )
               __UpperCAmelCase			:				Union[str, Any]													=    tf_tokenizer(
                   text=[sentence[0] for sentence in self.paired_sentences]							,				text_pair=[sentence[1] for sentence in self.paired_sentences]							,				)
               for key in merged_outputs.keys():
                    self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]							,				tf.intaa    ) == separated_outputs[key]    )    )

     @slow
     def 				lowerCamelCase__				(       self  :	str    )		->							Union[str, Any]:
          for tf_tokenizer in self.tf_tokenizers:
               __UpperCAmelCase			:				Optional[int]													=    tf.function(snake_case    )
               for test_inputs in (self.test_sentences, self.paired_sentences):
                    __UpperCAmelCase			:				int													=    tf.constant(snake_case    )
                    __UpperCAmelCase			:				Tuple													=    compiled_tokenizer(snake_case    )
                    __UpperCAmelCase			:				Optional[int]													=    tf_tokenizer(snake_case    )

                    for key in eager_outputs.keys():
                         self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]    )    )






     @slow
     def 				lowerCamelCase__				(       self  :	str    )		->							str:
          for tf_tokenizer in self.tf_tokenizers:
               __UpperCAmelCase			:				List[Any]													=    ModelToSave(tokenizer=snake_case    )
               __UpperCAmelCase			:				Union[str, Any]													=    tf.convert_to_tensor(self.test_sentences    )
               __UpperCAmelCase			:				Tuple													=    model(snake_case    )  # Build model with some sample inputs
               with TemporaryDirectory() as tempdir:
                    __UpperCAmelCase			:				Any													=    Path(snake_case    ) / '''saved.model'''
                    model.save(snake_case    )
                    __UpperCAmelCase			:				str													=    tf.keras.models.load_model(snake_case    )
               __UpperCAmelCase			:				Optional[int]													=    loaded_model(snake_case    )
               # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
               self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output    )    )							,				1E-5    )