"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowercase		:		str														=							logging.get_logger(__name__)

__lowercase		:		Union[str, Any]														=							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowercase		:		List[str]														=							{
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

__lowercase		:		str														=							{
    "bert-base-uncased": 512,
    "bert-large-uncased": 512,
    "bert-base-cased": 512,
    "bert-large-cased": 512,
    "bert-base-multilingual-uncased": 512,
    "bert-base-multilingual-cased": 512,
    "bert-base-chinese": 512,
    "bert-base-german-cased": 512,
    "bert-large-uncased-whole-word-masking": 512,
    "bert-large-cased-whole-word-masking": 512,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
    "bert-base-cased-finetuned-mrpc": 512,
    "bert-base-german-dbmdz-cased": 512,
    "bert-base-german-dbmdz-uncased": 512,
    "TurkuNLP/bert-base-finnish-cased-v1": 512,
    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
    "wietsedv/bert-base-dutch-cased": 512,
}

__lowercase		:		Union[str, Any]														=							{
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}



class      _A	(						_UpperCAmelCase					):




    """simple docstring"""




    UpperCamelCase_	:					Tuple							=					VOCAB_FILES_NAMES
    UpperCamelCase_	:					str							=					PRETRAINED_VOCAB_FILES_MAP
    UpperCamelCase_	:					List[Any]							=					PRETRAINED_INIT_CONFIGURATION
    UpperCamelCase_	:					List[str]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCamelCase_	:					Tuple							=					BertTokenizer
    def __init__(		self							:     List[str]	,      A_							:     int=None	,      A_							:     Tuple=None	,      A_							:     str=True	,      A_							:     Union[str, Any]="[UNK]"	,      A_							:     int="[SEP]"	,      A_							:     Dict="[PAD]"	,      A_							:     List[str]="[CLS]"	,      A_							:     Dict="[MASK]"	,      A_							:     Optional[int]=True	,      A_							:     List[Any]=None	,      **A_							:     str	,      )    ->       Optional[Any]:
         super().__init__(
             A_	,      tokenizer_file=A_	,      do_lower_case=A_	,      unk_token=A_	,      sep_token=A_	,      pad_token=A_	,      cls_token=A_	,      mask_token=A_	,      tokenize_chinese_chars=A_	,      strip_accents=A_	,      **A_	,      )

         __snake_case				=							json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
         if (
             normalizer_state.get('''lowercase'''	,      A_    ) != do_lower_case
             or normalizer_state.get('''strip_accents'''	,      A_    ) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''	,      A_    ) != tokenize_chinese_chars
         ):
              __snake_case				=							getattr(A_	,      normalizer_state.pop('''type'''    )    )
              __snake_case				=							do_lower_case
              __snake_case				=							strip_accents
              __snake_case				=							tokenize_chinese_chars
              __snake_case				=							normalizer_class(**A_    )

         __snake_case				=							do_lower_case
    def  lowercase					(		self							:     Dict	,      A_							:     int	,      A_							:     Dict=None    )    ->       Union[str, Any]:
         __snake_case				=							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output
    def  lowercase					(		self							:     List[str]	,      A_							:     List[int]	,      A_							:     Optional[List[int]] = None    )    ->       List[int]:
         __snake_case				=							[self.sep_token_id]
         __snake_case				=							[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep    ) * [0]
         return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







    def  lowercase					(		self							:     Dict	,      A_							:     str	,      A_							:     Optional[str] = None    )    ->       Tuple[str]:
         __snake_case				=							self._tokenizer.model.save(A_	,      name=A_    )
         return tuple(A_    )
"""simple docstring"""



import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
       from PIL import Image

       from transformers import (
           AutoProcessor,
           BertTokenizerFast,
           BlipImageProcessor,
           GPTaTokenizer,
           InstructBlipProcessor,
           PreTrainedTokenizerFast,
       )



@require_vision
class      _A	(						unittest.TestCase					):




    """simple docstring"""




    def  lowercase					(		self							:     str    )    ->       str:
         __snake_case				=							tempfile.mkdtemp()

         __snake_case				=							BlipImageProcessor()
         __snake_case				=							GPTaTokenizer.from_pretrained('''hf-internal-testing/tiny-random-GPT2Model'''    )
         __snake_case				=							BertTokenizerFast.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )

         __snake_case				=							InstructBlipProcessor(A_	,      A_	,      A_    )

         processor.save_pretrained(self.tmpdirname    )
    def  lowercase					(		self							:     Tuple	,      **A_							:     str    )    ->       List[Any]:
         return AutoProcessor.from_pretrained(self.tmpdirname	,      **A_    ).tokenizer
    def  lowercase					(		self							:     Union[str, Any]	,      **A_							:     Tuple    )    ->       Union[str, Any]:
         return AutoProcessor.from_pretrained(self.tmpdirname	,      **A_    ).image_processor
    def  lowercase					(		self							:     Union[str, Any]	,      **A_							:     Tuple    )    ->       Dict:
         return AutoProcessor.from_pretrained(self.tmpdirname	,      **A_    ).qformer_tokenizer
    def  lowercase					(		self							:     Optional[int]    )    ->       Optional[Any]:
         shutil.rmtree(self.tmpdirname    )
    def  lowercase					(		self							:     Optional[int]    )    ->       Tuple:
         __snake_case				=							[np.random.randint(255	,      size=(3, 30, 400)	,      dtype=np.uinta    )]

         __snake_case				=							[Image.fromarray(np.moveaxis(A_	,      0	,      -1    )    ) for x in image_inputs]

         return image_inputs
    def  lowercase					(		self							:     int    )    ->       Dict:
         __snake_case				=							InstructBlipProcessor(
             tokenizer=self.get_tokenizer()	,      image_processor=self.get_image_processor()	,      qformer_tokenizer=self.get_qformer_tokenizer()	,      )
         processor.save_pretrained(self.tmpdirname    )

         __snake_case				=							self.get_tokenizer(bos_token='''(BOS)'''	,      eos_token='''(EOS)'''    )
         __snake_case				=							self.get_image_processor(do_normalize=A_	,      padding_value=1.0    )

         __snake_case				=							InstructBlipProcessor.from_pretrained(
             self.tmpdirname	,      bos_token='''(BOS)'''	,      eos_token='''(EOS)'''	,      do_normalize=A_	,      padding_value=1.0    )

         self.assertEqual(processor.tokenizer.get_vocab()	,      tokenizer_add_kwargs.get_vocab()    )
         self.assertIsInstance(processor.tokenizer	,      A_    )

         self.assertEqual(processor.image_processor.to_json_string()	,      image_processor_add_kwargs.to_json_string()    )
         self.assertIsInstance(processor.image_processor	,      A_    )
         self.assertIsInstance(processor.qformer_tokenizer	,      A_    )
    def  lowercase					(		self							:     int    )    ->       str:
         __snake_case				=							self.get_image_processor()
         __snake_case				=							self.get_tokenizer()
         __snake_case				=							self.get_qformer_tokenizer()

         __snake_case				=							InstructBlipProcessor(
             tokenizer=A_	,      image_processor=A_	,      qformer_tokenizer=A_    )

         __snake_case				=							self.prepare_image_inputs()

         __snake_case				=							image_processor(A_	,      return_tensors='''np'''    )
         __snake_case				=							processor(images=A_	,      return_tensors='''np'''    )

         for key in input_feat_extract.keys():
              self.assertAlmostEqual(input_feat_extract[key].sum()	,      input_processor[key].sum()	,      delta=1E-2    )
    def  lowercase					(		self							:     List[str]    )    ->       Optional[int]:
         __snake_case				=							self.get_image_processor()
         __snake_case				=							self.get_tokenizer()
         __snake_case				=							self.get_qformer_tokenizer()

         __snake_case				=							InstructBlipProcessor(
             tokenizer=A_	,      image_processor=A_	,      qformer_tokenizer=A_    )

         __snake_case				=							'''lower newer'''

         __snake_case				=							processor(text=A_    )

         __snake_case				=							tokenizer(A_	,      return_token_type_ids=A_    )
         __snake_case				=							qformer_tokenizer(A_	,      return_token_type_ids=A_    )

         for key in encoded_tokens.keys():
              self.assertListEqual(encoded_tokens[key]	,      encoded_processor[key]    )

         for key in encoded_tokens_qformer.keys():
              self.assertListEqual(encoded_tokens_qformer[key]	,      encoded_processor['''qformer_''' + key]    )
    def  lowercase					(		self							:     List[str]    )    ->       int:
         __snake_case				=							self.get_image_processor()
         __snake_case				=							self.get_tokenizer()
         __snake_case				=							self.get_qformer_tokenizer()

         __snake_case				=							InstructBlipProcessor(
             tokenizer=A_	,      image_processor=A_	,      qformer_tokenizer=A_    )

         __snake_case				=							'''lower newer'''
         __snake_case				=							self.prepare_image_inputs()

         __snake_case				=							processor(text=A_	,      images=A_    )

         self.assertListEqual(
             list(inputs.keys()    )	,      ['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']	,      )

         # test if it raises when no input is passed
         with pytest.raises(A_    ):
              processor()
    def  lowercase					(		self							:     str    )    ->       Union[str, Any]:
         __snake_case				=							self.get_image_processor()
         __snake_case				=							self.get_tokenizer()
         __snake_case				=							self.get_qformer_tokenizer()

         __snake_case				=							InstructBlipProcessor(
             tokenizer=A_	,      image_processor=A_	,      qformer_tokenizer=A_    )

         __snake_case				=							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

         __snake_case				=							processor.batch_decode(A_    )
         __snake_case				=							tokenizer.batch_decode(A_    )

         self.assertListEqual(A_	,      A_    )







    def  lowercase					(		self							:     int    )    ->       List[str]:
         __snake_case				=							self.get_image_processor()
         __snake_case				=							self.get_tokenizer()
         __snake_case				=							self.get_qformer_tokenizer()

         __snake_case				=							InstructBlipProcessor(
             tokenizer=A_	,      image_processor=A_	,      qformer_tokenizer=A_    )

         __snake_case				=							'''lower newer'''
         __snake_case				=							self.prepare_image_inputs()

         __snake_case				=							processor(text=A_	,      images=A_    )

         self.assertListEqual(
             list(inputs.keys()    )	,      ['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']	,      )