

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowerCAmelCase     :   Any						     =  {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase     :   str						     =  {
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase     :   Dict						     =  {
    'unc-nlp/lxmert-base-uncased': 5_12,
}

lowerCAmelCase     :   List[str]						     =  {
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}
class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      List[Any]       =  VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :      List[str]       =  PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :      int       =  PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :      int       =  LxmertTokenizer

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE="[UNK]" ,					_SCREAMING_SNAKE_CASE="[SEP]" ,					_SCREAMING_SNAKE_CASE="[PAD]" ,					_SCREAMING_SNAKE_CASE="[CLS]" ,					_SCREAMING_SNAKE_CASE="[MASK]" ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=None ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        super().__init__(
            _SCREAMING_SNAKE_CASE ,					tokenizer_file=_SCREAMING_SNAKE_CASE ,					do_lower_case=_SCREAMING_SNAKE_CASE ,					unk_token=_SCREAMING_SNAKE_CASE ,					sep_token=_SCREAMING_SNAKE_CASE ,					pad_token=_SCREAMING_SNAKE_CASE ,					cls_token=_SCREAMING_SNAKE_CASE ,					mask_token=_SCREAMING_SNAKE_CASE ,					tokenize_chinese_chars=_SCREAMING_SNAKE_CASE ,					strip_accents=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE ,					)

        SCREAMING_SNAKE_CASE_				:  int           =     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
        if (
            normalizer_state.get('lowercase' ,					_SCREAMING_SNAKE_CASE					) != do_lower_case
            or normalizer_state.get('strip_accents' ,					_SCREAMING_SNAKE_CASE					) != strip_accents
            or normalizer_state.get('handle_chinese_chars' ,					_SCREAMING_SNAKE_CASE					) != tokenize_chinese_chars
        ):
              SCREAMING_SNAKE_CASE_				:  Any           =     getattr(_SCREAMING_SNAKE_CASE ,					normalizer_state.pop('type'					)					)
              SCREAMING_SNAKE_CASE_				:  List[Any]           =     do_lower_case
              SCREAMING_SNAKE_CASE_				:  List[str]           =     strip_accents
              SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tokenize_chinese_chars
              SCREAMING_SNAKE_CASE_				:  int           =     normalizer_class(**_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     do_lower_case

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE=None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     [self.sep_token_id]
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     [self.cls_token_id]
        if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
        return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     self._tokenizer.model.save(_SCREAMING_SNAKE_CASE ,					name=_SCREAMING_SNAKE_CASE					)
        return tuple(_SCREAMING_SNAKE_CASE					)









import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
    import torch

    from transformers import AutoModelForImageClassification

if is_vision_available():
    from transformers import AutoImageProcessor
@require_torch
@require_vision
class  _A		(  unittest.TestCase):

  @slow
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     AutoImageProcessor.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'					)
        SCREAMING_SNAKE_CASE_				:  Any           =     AutoModelForImageClassification.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'					)
        model.to(_SCREAMING_SNAKE_CASE					)

        from datasets import load_dataset

        SCREAMING_SNAKE_CASE_				:  str           =     load_dataset('nielsr/rvlcdip-demo'					)

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     dataset['train'][0]['image'].convert('RGB'					)

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     image_processor(_SCREAMING_SNAKE_CASE ,					return_tensors='pt'					).to(_SCREAMING_SNAKE_CASE					)

        # forward pass
        with torch.no_grad():
              SCREAMING_SNAKE_CASE_				:  int           =     model(**_SCREAMING_SNAKE_CASE					)
              SCREAMING_SNAKE_CASE_				:  Any           =     outputs.logits

        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     torch.Size((1, 16)					)
        self.assertEqual(logits.shape ,					_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  List[str]           =     torch.tensor(
            [-0.4158, -0.4092, -0.4347] ,					device=_SCREAMING_SNAKE_CASE ,					dtype=torch.float ,					)
        self.assertTrue(torch.allclose(logits[0, :3] ,					_SCREAMING_SNAKE_CASE ,					atol=1e-4					)					)






