


'''simple docstring'''


import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY







@is_pipeline_test
class 							__magic_name__	(       unittest.TestCase ):

     """simple docstring"""






     lowerCamelCase__       						=				MODEL_FOR_MASKED_LM_MAPPING
     lowerCamelCase__       						=				TF_MODEL_FOR_MASKED_LM_MAPPING


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          super().tearDown()
          # clean-up as much as possible GPU memory occupied by PyTorch
          gc.collect()
          if is_torch_available():
               import torch

               torch.cuda.empty_cache()


     @require_tf
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       str      =							pipeline(task="fill-mask"			,     model="sshleifer/tiny-distilroberta-base"			,     top_k=2			,     framework="tf"		)
          __A	:       Optional[int]      =							unmasker("My name is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {"sequence": "My name is grouped", "score": 2.1E-05, "token": 3_8015, "token_str": " grouped"},
                  {"sequence": "My name is accuser", "score": 2.1E-05, "token": 2_5506, "token_str": " accuser"},
              ]			,     )

          __A	:       int      =							unmasker("The largest city in France is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {
                      "sequence": "The largest city in France is grouped",
                      "score": 2.1E-05,
                      "token": 3_8015,
                      "token_str": " grouped",
                  },
                  {
                      "sequence": "The largest city in France is accuser",
                      "score": 2.1E-05,
                      "token": 2_5506,
                      "token_str": " accuser",
                  },
              ]			,     )

          __A	:       Dict      =							unmasker("My name is <mask>"			,     targets=[" Patrick", " Clara", " Teven"]			,     top_k=3		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {"sequence": "My name is Clara", "score": 2E-05, "token": 1_3606, "token_str": " Clara"},
                  {"sequence": "My name is Patrick", "score": 2E-05, "token": 3499, "token_str": " Patrick"},
                  {"sequence": "My name is Te", "score": 1.9E-05, "token": 2941, "token_str": " Te"},
              ]			,     )


     @require_torch
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       List[Any]      =							pipeline(task="fill-mask"			,     model="sshleifer/tiny-distilroberta-base"			,     top_k=2			,     framework="pt"		)

          __A	:       Union[str, Any]      =							unmasker("My name is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {"sequence": "My name is Maul", "score": 2.2E-05, "token": 3_5676, "token_str": " Maul"},
                  {"sequence": "My name isELS", "score": 2.2E-05, "token": 1_6416, "token_str": "ELS"},
              ]			,     )

          __A	:       Tuple      =							unmasker("The largest city in France is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {
                      "sequence": "The largest city in France is Maul",
                      "score": 2.2E-05,
                      "token": 3_5676,
                      "token_str": " Maul",
                  },
                  {"sequence": "The largest city in France isELS", "score": 2.2E-05, "token": 1_6416, "token_str": "ELS"},
              ]			,     )

          __A	:       int      =							unmasker("My name is <mask>"			,     targets=[" Patrick", " Clara", " Teven"]			,     top_k=3		)
          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  {"sequence": "My name is Patrick", "score": 2.1E-05, "token": 3499, "token_str": " Patrick"},
                  {"sequence": "My name is Te", "score": 2E-05, "token": 2941, "token_str": " Te"},
                  {"sequence": "My name is Clara", "score": 2E-05, "token": 1_3606, "token_str": " Clara"},
              ]			,     )

          __A	:       Tuple      =							unmasker("My name is <mask> <mask>"			,     top_k=2		)

          self.assertEqual(
              nested_simplify(lowerCamelCase			,     decimals=6		)			,     [
                  [
                      {
                          "score": 2.2E-05,
                          "token": 3_5676,
                          "token_str": " Maul",
                          "sequence": "<s>My name is Maul<mask></s>",
                      },
                      {"score": 2.2E-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
                  ],
                  [
                      {
                          "score": 2.2E-05,
                          "token": 3_5676,
                          "token_str": " Maul",
                          "sequence": "<s>My name is<mask> Maul</s>",
                      },
                      {"score": 2.2E-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
                  ],
              ]			,     )


     @require_torch_gpu
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Any      =							pipeline("fill-mask"			,     model="hf-internal-testing/tiny-random-distilbert"			,     device=0			,     framework="pt"		)

          # convert model to fp16
          pipe.model.half()

          __A	:       Dict      =							pipe("Paris is the [MASK] of France."		)
          # We actually don't care about the result, we just want to make sure
          # it works, meaning the float16 tensor got casted back to float32
          # for postprocessing.
          self.assertIsInstance(lowerCamelCase			,     lowerCamelCase		)


     @slow
     @require_torch
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[int]      =							pipeline(task="fill-mask"			,     model="distilroberta-base"			,     top_k=2			,     framework="pt"		)
          self.run_large_test(lowerCamelCase		)


     @slow
     @require_tf
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							pipeline(task="fill-mask"			,     model="distilroberta-base"			,     top_k=2			,     framework="tf"		)
          self.run_large_test(lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       List[Any]      =							unmasker("My name is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase		)			,     [
                  {"sequence": "My name is John", "score": 0.008, "token": 610, "token_str": " John"},
                  {"sequence": "My name is Chris", "score": 0.007, "token": 1573, "token_str": " Chris"},
              ]			,     )
          __A	:       Any      =							unmasker("The largest city in France is <mask>"		)
          self.assertEqual(
              nested_simplify(lowerCamelCase		)			,     [
                  {
                      "sequence": "The largest city in France is Paris",
                      "score": 0.251,
                      "token": 2201,
                      "token_str": " Paris",
                  },
                  {
                      "sequence": "The largest city in France is Lyon",
                      "score": 0.214,
                      "token": 1_2790,
                      "token_str": " Lyon",
                  },
              ]			,     )

          __A	:       Any      =							unmasker("My name is <mask>"			,     targets=[" Patrick", " Clara", " Teven"]			,     top_k=3		)
          self.assertEqual(
              nested_simplify(lowerCamelCase		)			,     [
                  {"sequence": "My name is Patrick", "score": 0.005, "token": 3499, "token_str": " Patrick"},
                  {"sequence": "My name is Clara", "score": 0.000, "token": 1_3606, "token_str": " Clara"},
                  {"sequence": "My name is Te", "score": 0.000, "token": 2941, "token_str": " Te"},
              ]			,     )


     @require_torch
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							pipeline(task="fill-mask"			,     model="sshleifer/tiny-distilroberta-base"			,     framework="pt"		)
          __A	:       Union[str, Any]      =							None
          __A	:       Optional[Any]      =							None
          self.run_pipeline_test(lowerCamelCase			,     []		)


     @require_tf
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							pipeline(task="fill-mask"			,     model="sshleifer/tiny-distilroberta-base"			,     framework="tf"		)
          __A	:       Optional[int]      =							None
          __A	:       List[str]      =							None
          self.run_pipeline_test(lowerCamelCase			,     []		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          if tokenizer is None or tokenizer.mask_token_id is None:
               self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)"		)

          __A	:       Optional[int]      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)
          __A	:       List[Any]      =							[
              f"This is another {tokenizer.mask_token} test",
          ]
          return fill_masker, examples


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							fill_masker.tokenizer
          __A	:       List[str]      =							fill_masker.model

          __A	:       Dict      =							fill_masker(
              f"This is a {tokenizer.mask_token}"			,     )
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )

          __A	:       int      =							fill_masker([f"This is a {tokenizer.mask_token}"]		)
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )

          __A	:       Dict      =							fill_masker([f"This is a {tokenizer.mask_token}", f"Another {tokenizer.mask_token} great test."]		)
          self.assertEqual(
              lowerCamelCase			,     [
                  [
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  ],
                  [
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  ],
              ]			,     )

          with self.assertRaises(lowerCamelCase		):
               fill_masker([None]		)
          # No mask_token is not supported
          with self.assertRaises(lowerCamelCase		):
               fill_masker("This is"		)

          self.run_test_top_k(lowerCamelCase			,     lowerCamelCase		)
          self.run_test_targets(lowerCamelCase			,     lowerCamelCase		)
          self.run_test_top_k_targets(lowerCamelCase			,     lowerCamelCase		)
          self.fill_mask_with_duplicate_targets_and_top_k(lowerCamelCase			,     lowerCamelCase		)
          self.fill_mask_with_multiple_masks(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Any      =							tokenizer.get_vocab()
          __A	:       Tuple      =							sorted(vocab.keys()		)[:2]
          # Pipeline argument
          __A	:       int      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase			,     targets=lowerCamelCase		)
          __A	:       List[str]      =							fill_masker(f"This is a {tokenizer.mask_token}"		)
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )
          __A	:       Any      =							{vocab[el] for el in targets}
          self.assertEqual({el["token"] for el in outputs}			,     lowerCamelCase		)
          __A	:       str      =							[tokenizer.decode([x]		) for x in target_ids]
          self.assertEqual({el["token_str"] for el in outputs}			,     set(lowerCamelCase		)		)

          # Call argument
          __A	:       List[Any]      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)
          __A	:       Optional[int]      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=lowerCamelCase		)
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )
          __A	:       Union[str, Any]      =							{vocab[el] for el in targets}
          self.assertEqual({el["token"] for el in outputs}			,     lowerCamelCase		)
          __A	:       Dict      =							[tokenizer.decode([x]		) for x in target_ids]
          self.assertEqual({el["token_str"] for el in outputs}			,     set(lowerCamelCase		)		)

          # Score equivalence
          __A	:       str      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=lowerCamelCase		)
          __A	:       Any      =							[top_mask["token_str"] for top_mask in outputs]
          __A	:       Any      =							[top_mask["score"] for top_mask in outputs]

          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(lowerCamelCase		) == set(lowerCamelCase		):
               __A	:       str      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=lowerCamelCase		)
               __A	:       List[str]      =							[top_mask["score"] for top_mask in unmasked_targets]
               self.assertEqual(nested_simplify(lowerCamelCase		)			,     nested_simplify(lowerCamelCase		)		)

          # Raises with invalid
          with self.assertRaises(lowerCamelCase		):
               __A	:       int      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=[]		)
          # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
          if "" not in tokenizer.get_vocab():
               with self.assertRaises(lowerCamelCase		):
                    __A	:       Optional[int]      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=[""]		)
               with self.assertRaises(lowerCamelCase		):
                    __A	:       int      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     targets=""		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Tuple      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase			,     top_k=2		)
          __A	:       Dict      =							fill_masker(f"This is a {tokenizer.mask_token}"		)
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )

          __A	:       Optional[int]      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)
          __A	:       Optional[Any]      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     top_k=2		)
          self.assertEqual(
              lowerCamelCase			,     [
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
              ]			,     )
          self.assertEqual(nested_simplify(lowerCamelCase		)			,     nested_simplify(lowerCamelCase		)		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Dict      =							tokenizer.get_vocab()
          __A	:       Optional[Any]      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)

          # top_k=2, ntargets=3
          __A	:       Optional[int]      =							sorted(vocab.keys()		)[:3]
          __A	:       Union[str, Any]      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     top_k=2			,     targets=lowerCamelCase		)

          # If we use the most probably targets, and filter differently, we should still
          # have the same results
          __A	:       Any      =							[el["token_str"] for el in sorted(lowerCamelCase			,     key=lambda lowerCamelCase		: x["score"]			,     reverse=lowerCamelCase		)]
          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(lowerCamelCase		).issubset(lowerCamelCase		):
               __A	:       Optional[Any]      =							fill_masker(f"This is a {tokenizer.mask_token}"			,     top_k=3			,     targets=lowerCamelCase		)
               # They should yield exactly the same result
               self.assertEqual(nested_simplify(lowerCamelCase		)			,     nested_simplify(lowerCamelCase		)		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Any      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)
          __A	:       Any      =							tokenizer.get_vocab()
          # String duplicates + id duplicates
          __A	:       Any      =							sorted(vocab.keys()		)[:3]
          __A	:       int      =							[targets[0], targets[1], targets[0], targets[2], targets[1]]
          __A	:       Union[str, Any]      =							fill_masker(f"My name is {tokenizer.mask_token}"			,     targets=lowerCamelCase			,     top_k=10		)

          # The target list contains duplicates, so we can't output more
          # than them
          self.assertEqual(len(lowerCamelCase		)			,     3		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Tuple      =							FillMaskPipeline(model=lowerCamelCase			,     tokenizer=lowerCamelCase		)

          __A	:       List[Any]      =							fill_masker(
              f"This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"			,     top_k=2		)
          self.assertEqual(
              lowerCamelCase			,     [
                  [
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  ],
                  [
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  ],
                  [
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                      {"sequence": ANY(lowerCamelCase		), "score": ANY(lowerCamelCase		), "token": ANY(lowerCamelCase		), "token_str": ANY(lowerCamelCase		)},
                  ],
              ]			,     )





'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
      from PIL import Image

      from transformers import AlignProcessor, EfficientNetImageProcessor







@require_vision
class 							__magic_name__	(       unittest.TestCase ):

     """simple docstring"""






     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							tempfile.mkdtemp()

          __A	:       List[Any]      =							[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          __A	:       Tuple      =							os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]		)
          with open(self.vocab_file			,     "w"			,     encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

          __A	:       List[str]      =							{
              "do_resize": True,
              "size": 20,
              "do_center_crop": True,
              "crop_size": 18,
              "do_normalize": True,
              "image_mean": [0.4814_5466, 0.457_8275, 0.4082_1073],
              "image_std": [0.2686_2954, 0.2613_0258, 0.2757_7711],
          }
          __A	:       Union[str, Any]      =							os.path.join(self.tmpdirname			,     lowerCamelCase		)
          with open(self.image_processor_file			,     "w"			,     encoding="utf-8"		) as fp:
               json.dump(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return BertTokenizer.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return BertTokenizerFast.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     **lowerCamelCase		):


          '''simple docstring'''





          return EfficientNetImageProcessor.from_pretrained(self.tmpdirname			,     **lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          shutil.rmtree(self.tmpdirname		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       int      =							[np.random.randint(255			,     size=(3, 30, 400)			,     dtype=np.uinta		)]
          __A	:       int      =							[Image.fromarray(np.moveaxis(lowerCamelCase			,     0			,     -1		)		) for x in image_inputs]
          return image_inputs


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Union[str, Any]      =							self.get_tokenizer()
          __A	:       int      =							self.get_rust_tokenizer()
          __A	:       List[str]      =							self.get_image_processor()

          __A	:       List[Any]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)
          processor_slow.save_pretrained(self.tmpdirname		)
          __A	:       List[Any]      =							AlignProcessor.from_pretrained(self.tmpdirname			,     use_fast=lowerCamelCase		)

          __A	:       int      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)
          processor_fast.save_pretrained(self.tmpdirname		)
          __A	:       List[str]      =							AlignProcessor.from_pretrained(self.tmpdirname		)

          self.assertEqual(processor_slow.tokenizer.get_vocab()			,     tokenizer_slow.get_vocab()		)
          self.assertEqual(processor_fast.tokenizer.get_vocab()			,     tokenizer_fast.get_vocab()		)
          self.assertEqual(tokenizer_slow.get_vocab()			,     tokenizer_fast.get_vocab()		)
          self.assertIsInstance(processor_slow.tokenizer			,     lowerCamelCase		)
          self.assertIsInstance(processor_fast.tokenizer			,     lowerCamelCase		)

          self.assertEqual(processor_slow.image_processor.to_json_string()			,     image_processor.to_json_string()		)
          self.assertEqual(processor_fast.image_processor.to_json_string()			,     image_processor.to_json_string()		)
          self.assertIsInstance(processor_slow.image_processor			,     lowerCamelCase		)
          self.assertIsInstance(processor_fast.image_processor			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       List[Any]      =							AlignProcessor(tokenizer=self.get_tokenizer()			,     image_processor=self.get_image_processor()		)
          processor.save_pretrained(self.tmpdirname		)

          __A	:       List[str]      =							self.get_tokenizer(bos_token="(BOS)"			,     eos_token="(EOS)"		)
          __A	:       int      =							self.get_image_processor(do_normalize=lowerCamelCase			,     padding_value=1.0		)

          __A	:       Optional[int]      =							AlignProcessor.from_pretrained(
              self.tmpdirname			,     bos_token="(BOS)"			,     eos_token="(EOS)"			,     do_normalize=lowerCamelCase			,     padding_value=1.0		)

          self.assertEqual(processor.tokenizer.get_vocab()			,     tokenizer_add_kwargs.get_vocab()		)
          self.assertIsInstance(processor.tokenizer			,     lowerCamelCase		)

          self.assertEqual(processor.image_processor.to_json_string()			,     image_processor_add_kwargs.to_json_string()		)
          self.assertIsInstance(processor.image_processor			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							self.get_image_processor()
          __A	:       Optional[Any]      =							self.get_tokenizer()

          __A	:       List[str]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       int      =							self.prepare_image_inputs()

          __A	:       Optional[int]      =							image_processor(lowerCamelCase			,     return_tensors="np"		)
          __A	:       Dict      =							processor(images=lowerCamelCase			,     return_tensors="np"		)

          for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum()			,     input_processor[key].sum()			,     delta=1E-2		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       str      =							self.get_image_processor()
          __A	:       int      =							self.get_tokenizer()

          __A	:       Optional[int]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Optional[Any]      =							"lower newer"

          __A	:       Union[str, Any]      =							processor(text=lowerCamelCase		)

          __A	:       int      =							tokenizer(lowerCamelCase			,     padding="max_length"			,     max_length=64		)

          for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]			,     encoded_processor[key]		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[int]      =							self.get_image_processor()
          __A	:       Optional[Any]      =							self.get_tokenizer()

          __A	:       int      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Tuple      =							"lower newer"
          __A	:       int      =							self.prepare_image_inputs()

          __A	:       str      =							processor(text=lowerCamelCase			,     images=lowerCamelCase		)

          self.assertListEqual(list(inputs.keys()		)			,     ["input_ids", "token_type_ids", "attention_mask", "pixel_values"]		)

          # test if it raises when no input is passed
          with pytest.raises(lowerCamelCase		):
               processor()


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Dict      =							self.get_image_processor()
          __A	:       Tuple      =							self.get_tokenizer()

          __A	:       Dict      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       Optional[int]      =							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          __A	:       Optional[int]      =							processor.batch_decode(lowerCamelCase		)
          __A	:       Optional[int]      =							tokenizer.batch_decode(lowerCamelCase		)

          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       str      =							self.get_image_processor()
          __A	:       Union[str, Any]      =							self.get_tokenizer()

          __A	:       Union[str, Any]      =							AlignProcessor(tokenizer=lowerCamelCase			,     image_processor=lowerCamelCase		)

          __A	:       int      =							"lower newer"
          __A	:       str      =							self.prepare_image_inputs()

          __A	:       Tuple      =							processor(text=lowerCamelCase			,     images=lowerCamelCase		)

          self.assertListEqual(list(inputs.keys()		)			,     processor.model_input_names		)

