




'''simple docstring'''







# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
      import torch







class 					_a (TensorFormatter[Mapping, """torch.Tensor""", Mapping]	):





     '''simple docstring'''



     def __init__(				self							,__a=None							,**__a							)      ->				Any:
          super().__init__(features=__a							)
          snake_case			:      int        =			torch_tensor_kwargs
          import torch  # noqa import torch at initialization







     def 			snake_case_      (				self							,__a							)      ->				Any:
          import torch

          if isinstance(__a							,__a							) and column:
               if all(
                   isinstance(__a							,torch.Tensor							) and x.shape == column[0].shape and x.dtype == column[0].dtype
                   for x in column							):
                    return torch.stack(__a							)
          return column







     def 			snake_case_      (				self							,__a							)      ->				Tuple:
          import torch

          if isinstance(__a							,(str, bytes, type(__a							))							):
               return value
          elif isinstance(__a							,(np.character, np.ndarray)							) and np.issubdtype(value.dtype							,np.character							):
               return value.tolist()

          snake_case			:      List[str]        =			{}

          if isinstance(__a							,(np.number, np.ndarray)							) and np.issubdtype(value.dtype							,np.integer							):
               snake_case			:      Union[str, Any]        =			{"""dtype""": torch.intaa}
          elif isinstance(__a							,(np.number, np.ndarray)							) and np.issubdtype(value.dtype							,np.floating							):
               snake_case			:      List[str]        =			{"""dtype""": torch.floataa}
          elif config.PIL_AVAILABLE and "PIL" in sys.modules:
               import PIL.Image

               if isinstance(__a							,PIL.Image.Image							):
                    snake_case			:      Any        =			np.asarray(__a							)
          return torch.tensor(__a							,**{**default_dtype, **self.torch_tensor_kwargs}							)







     def 			snake_case_      (				self							,__a							)      ->				Tuple:
          import torch

          # support for torch, tf, jax etc.
          if hasattr(__a							,"""__array__"""							) and not isinstance(__a							,torch.Tensor							):
               snake_case			:      Union[str, Any]        =			data_struct.__array__()
          # support for nested types like struct of list of struct
          if isinstance(__a							,np.ndarray							):
               if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
                    return self._consolidate([self.recursive_tensorize(__a							) for substruct in data_struct]							)
          elif isinstance(__a							,(list, tuple)							):
               return self._consolidate([self.recursive_tensorize(__a							) for substruct in data_struct]							)
          return self._tensorize(__a							)







     def 			snake_case_      (				self							,__a							)      ->				Optional[Any]:
          return map_nested(self._recursive_tensorize							,__a							,map_list=__a							)







     def 			snake_case_      (				self							,__a							)      ->				Mapping:
          snake_case			:      Any        =			self.numpy_arrow_extractor().extract_row(__a							)
          snake_case			:      Optional[int]        =			self.python_features_decoder.decode_row(__a							)
          return self.recursive_tensorize(__a							)







     def 			snake_case_      (				self							,__a							)      ->				"torch.Tensor":
          snake_case			:      Any        =			self.numpy_arrow_extractor().extract_column(__a							)
          snake_case			:      Any        =			self.python_features_decoder.decode_column(__a							,pa_table.column_names[0]							)
          snake_case			:      List[str]        =			self.recursive_tensorize(__a							)
          snake_case			:      Optional[Any]        =			self._consolidate(__a							)
          return column







     def 			snake_case_      (				self							,__a							)      ->				Mapping:
          snake_case			:      Tuple        =			self.numpy_arrow_extractor().extract_batch(__a							)
          snake_case			:      str        =			self.python_features_decoder.decode_batch(__a							)
          snake_case			:      Optional[Any]        =			self.recursive_tensorize(__a							)
          for column_name in batch:
               snake_case			:      Optional[int]        =			self._consolidate(batch[column_name]							)
          return batch









'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



