





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
       from PIL import Image

       from transformers import AlignProcessor, EfficientNetImageProcessor




@require_vision
class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		tempfile.mkdtemp()

        A__		   =		[
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        A__		   =		os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['''vocab_file''']					)
        with open(self.vocab_file,'''w''',encoding='''utf-8'''					) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)

        A__		   =		{
            '''do_resize''': True,
            '''size''': 20,
            '''do_center_crop''': True,
            '''crop_size''': 18,
            '''do_normalize''': True,
            '''image_mean''': [0.48145466, 0.4578275, 0.40821073],
            '''image_std''': [0.26862954, 0.26130258, 0.27577711],
        }
        A__		   =		os.path.join(self.tmpdirname,__lowerCamelCase					)
        with open(self.image_processor_file,'''w''',encoding='''utf-8'''					) as fp:
               json.dump(__lowerCamelCase,__lowerCamelCase					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return BertTokenizer.from_pretrained(self.tmpdirname,**__lowerCamelCase					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return BertTokenizerFast.from_pretrained(self.tmpdirname,**__lowerCamelCase					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return EfficientNetImageProcessor.from_pretrained(self.tmpdirname,**__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        shutil.rmtree(self.tmpdirname					)



 def 							UpperCamelCase     (				self					):
        A__		   =		[np.random.randint(255,size=(3, 30, 400),dtype=np.uinta					)]
        A__		   =		[Image.fromarray(np.moveaxis(__lowerCamelCase,0,-1					)					) for x in image_inputs]
        return image_inputs



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_tokenizer()
        A__		   =		self.get_rust_tokenizer()
        A__		   =		self.get_image_processor()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)
        processor_slow.save_pretrained(self.tmpdirname					)
        A__		   =		AlignProcessor.from_pretrained(self.tmpdirname,use_fast=__lowerCamelCase					)

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)
        processor_fast.save_pretrained(self.tmpdirname					)
        A__		   =		AlignProcessor.from_pretrained(self.tmpdirname					)

        self.assertEqual(processor_slow.tokenizer.get_vocab(),tokenizer_slow.get_vocab()					)
        self.assertEqual(processor_fast.tokenizer.get_vocab(),tokenizer_fast.get_vocab()					)
        self.assertEqual(tokenizer_slow.get_vocab(),tokenizer_fast.get_vocab()					)
        self.assertIsInstance(processor_slow.tokenizer,__lowerCamelCase					)
        self.assertIsInstance(processor_fast.tokenizer,__lowerCamelCase					)

        self.assertEqual(processor_slow.image_processor.to_json_string(),image_processor.to_json_string()					)
        self.assertEqual(processor_fast.image_processor.to_json_string(),image_processor.to_json_string()					)
        self.assertIsInstance(processor_slow.image_processor,__lowerCamelCase					)
        self.assertIsInstance(processor_fast.image_processor,__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        A__		   =		AlignProcessor(tokenizer=self.get_tokenizer(),image_processor=self.get_image_processor()					)
        processor.save_pretrained(self.tmpdirname					)

        A__		   =		self.get_tokenizer(bos_token='''(BOS)''',eos_token='''(EOS)'''					)
        A__		   =		self.get_image_processor(do_normalize=__lowerCamelCase,padding_value=1.0					)

        A__		   =		AlignProcessor.from_pretrained(
            self.tmpdirname,bos_token='''(BOS)''',eos_token='''(EOS)''',do_normalize=__lowerCamelCase,padding_value=1.0					)

        self.assertEqual(processor.tokenizer.get_vocab(),tokenizer_add_kwargs.get_vocab()					)
        self.assertIsInstance(processor.tokenizer,__lowerCamelCase					)

        self.assertEqual(processor.image_processor.to_json_string(),image_processor_add_kwargs.to_json_string()					)
        self.assertIsInstance(processor.image_processor,__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()
        A__		   =		self.get_tokenizer()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)

        A__		   =		self.prepare_image_inputs()

        A__		   =		image_processor(__lowerCamelCase,return_tensors='''np'''					)
        A__		   =		processor(images=__lowerCamelCase,return_tensors='''np'''					)

        for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum(),input_processor[key].sum(),delta=1E-2					)



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()
        A__		   =		self.get_tokenizer()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)

        A__		   =		'''lower newer'''

        A__		   =		processor(text=__lowerCamelCase					)

        A__		   =		tokenizer(__lowerCamelCase,padding='''max_length''',max_length=64					)

        for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key],encoded_processor[key]					)



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()
        A__		   =		self.get_tokenizer()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)

        A__		   =		'''lower newer'''
        A__		   =		self.prepare_image_inputs()

        A__		   =		processor(text=__lowerCamelCase,images=__lowerCamelCase					)

        self.assertListEqual(list(inputs.keys()					),['''input_ids''', '''token_type_ids''', '''attention_mask''', '''pixel_values''']					)

        # test if it raises when no input is passed
        with pytest.raises(__lowerCamelCase					):
               processor()



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()
        A__		   =		self.get_tokenizer()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)

        A__		   =		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        A__		   =		processor.batch_decode(__lowerCamelCase					)
        A__		   =		tokenizer.batch_decode(__lowerCamelCase					)

        self.assertListEqual(__lowerCamelCase,__lowerCamelCase					)




 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()
        A__		   =		self.get_tokenizer()

        A__		   =		AlignProcessor(tokenizer=__lowerCamelCase,image_processor=__lowerCamelCase					)

        A__		   =		'''lower newer'''
        A__		   =		self.prepare_image_inputs()

        A__		   =		processor(text=__lowerCamelCase,images=__lowerCamelCase					)

        self.assertListEqual(list(inputs.keys()					),processor.model_input_names					)












import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


a__: Union[str, Any]							   =						'hf-internal-testing/tiny-random-bert'
a__: Optional[int]							   =						os.path.join(TRANSFORMERS_CACHE, 'models--hf-internal-testing--tiny-random-bert')
a__: List[str]							   =						'9b8c223d42b2188cb49d29af482996f9d0f3e5a6'




class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase					)
        # Should have downloaded the file in here
        self.assertTrue(os.path.isdir(__lowerCamelCase					)					)
        # Cache should contain at least those three subfolders:
        for subfolder in ["blobs", "refs", "snapshots"]:
               self.assertTrue(os.path.isdir(os.path.join(__lowerCamelCase,__lowerCamelCase					)					)					)
        with open(os.path.join(__lowerCamelCase,'''refs''','''main'''					)					) as f:
               A__		   =		f.read()
        self.assertEqual(__lowerCamelCase,os.path.join(__lowerCamelCase,'''snapshots''',__lowerCamelCase,__lowerCamelCase					)					)
        self.assertTrue(os.path.isfile(__lowerCamelCase					)					)

        # File is cached at the same place the second time.
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase					)
        self.assertEqual(__lowerCamelCase,__lowerCamelCase					)

        # Using a specific revision to test the full commit hash.
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase,revision='''9b8c223'''					)
        self.assertEqual(__lowerCamelCase,os.path.join(__lowerCamelCase,'''snapshots''',__lowerCamelCase,__lowerCamelCase					)					)



 def 							UpperCamelCase     (				self					):
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid model identifier'''					):
               A__		   =		cached_file('''tiny-random-bert''',__lowerCamelCase					)

        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid git identifier'''					):
               A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase,revision='''aaaa'''					)

        with self.assertRaisesRegex(__lowerCamelCase,'''does not appear to have a file named'''					):
               A__		   =		cached_file(__lowerCamelCase,'''conf'''					)



 def 							UpperCamelCase     (				self					):
        with self.assertRaisesRegex(__lowerCamelCase,'''does not appear to have a file named'''					):
               A__		   =		cached_file(__lowerCamelCase,'''conf'''					)

        with open(os.path.join(__lowerCamelCase,'''refs''','''main'''					)					) as f:
               A__		   =		f.read()
        self.assertTrue(os.path.isfile(os.path.join(__lowerCamelCase,'''.no_exist''',__lowerCamelCase,'''conf'''					)					)					)

        A__		   =		cached_file(__lowerCamelCase,'''conf''',_raise_exceptions_for_missing_entries=__lowerCamelCase					)
        self.assertIsNone(__lowerCamelCase					)

        A__		   =		cached_file(__lowerCamelCase,'''conf''',local_files_only=__lowerCamelCase,_raise_exceptions_for_missing_entries=__lowerCamelCase					)
        self.assertIsNone(__lowerCamelCase					)

        A__		   =		mock.Mock()
        A__		   =		500
        A__		   =		{}
        A__		   =		HTTPError
        A__		   =		{}

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request''',return_value=__lowerCamelCase					) as mock_head:
               A__		   =		cached_file(__lowerCamelCase,'''conf''',_raise_exceptions_for_connection_errors=__lowerCamelCase					)
               self.assertIsNone(__lowerCamelCase					)
               # This check we did call the fake head request
               mock_head.assert_called()



 def 							UpperCamelCase     (				self					):
        self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)
        self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)
        self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)



 def 							UpperCamelCase     (				self					):
        # `get_file_from_repo` returns None if the file does not exist
        self.assertIsNone(get_file_from_repo('''bert-base-cased''','''ahah.txt'''					)					)

        # The function raises if the repository does not exist.
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid model identifier'''					):
               get_file_from_repo('''bert-base-case''',__lowerCamelCase					)

        # The function raises if the revision does not exist.
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid git identifier'''					):
               get_file_from_repo('''bert-base-cased''',__lowerCamelCase,revision='''ahaha'''					)

        A__		   =		get_file_from_repo('''bert-base-cased''',__lowerCamelCase					)
        # The name is the cached name which is not very easy to test, so instead we load the content.
        A__		   =		json.loads(open(__lowerCamelCase,'''r'''					).read()					)
        self.assertEqual(config['''hidden_size'''],768					)




 def 							UpperCamelCase     (				self					):
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		Path(__lowerCamelCase					) / '''a.txt'''
               filename.touch()
               self.assertEqual(get_file_from_repo(__lowerCamelCase,'''a.txt'''					),str(__lowerCamelCase					)					)

               self.assertIsNone(get_file_from_repo(__lowerCamelCase,'''b.txt'''					)					)





