

'''simple docstring'''





import warnings

from transformers import AutoTokenizer
from transformers.utils import is_torch_available
from transformers.utils.generic import ExplicitEnum

from ...processing_utils import ProcessorMixin


if is_torch_available():
 import torch
class 				lowercase       (							a__      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							"char"
    lowerCAmelCase__      				=							"bpe"
    lowerCAmelCase__      				=							"wp"
__A							:							str  							=		(DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE)
class 				lowercase       (							a__      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							["image_processor", "char_tokenizer"]
    lowerCAmelCase__      				=							"ViTImageProcessor"
    lowerCAmelCase__      				=							"MgpstrTokenizer"

    def __init__(					self     :     List[str]				,     __lowerCamelCase     :     str=None				,     __lowerCamelCase     :     Optional[Any]=None				,     **__lowerCamelCase     :     Union[str, Any]     )							->							int:

     '''simple docstring'''




     lowerCamelCase__  =     None
     if "feature_extractor" in kwargs:
      warnings.warn(
          "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
          " instead."				,     _A				,     )
      lowerCamelCase__  =     kwargs.pop("feature_extractor"     )

     lowerCamelCase__  =     image_processor if image_processor is not None else feature_extractor
     if image_processor is None:
      raise ValueError("You need to specify an `image_processor`."     )
     if tokenizer is None:
      raise ValueError("You need to specify a `tokenizer`."     )

     lowerCamelCase__  =     tokenizer
     lowerCamelCase__  =     AutoTokenizer.from_pretrained("gpt2"     )
     lowerCamelCase__  =     AutoTokenizer.from_pretrained("bert-base-uncased"     )

     super().__init__(_A				,     _A     )

    def __call__(					self     :     Tuple				,     __lowerCamelCase     :     Dict=None				,     __lowerCamelCase     :     Optional[Any]=None				,     __lowerCamelCase     :     Dict=None				,     **__lowerCamelCase     :     Optional[Any]     )							->							str:

     '''simple docstring'''




     if images is None and text is None:
      raise ValueError("You need to specify either an `images` or `text` input to process."     )

     if images is not None:
      lowerCamelCase__  =     self.image_processor(_A				,     return_tensors=_A				,     **_A     )
     if text is not None:
      lowerCamelCase__  =     self.char_tokenizer(_A				,     return_tensors=_A				,     **_A     )

     if text is None:
      return inputs
     elif images is None:
      return encodings
     else:
      lowerCamelCase__  =     encodings['input_ids']
      return inputs

    def    a__				(					self     :     Optional[Any]				,     __lowerCamelCase     :     Union[str, Any]     )							->							Tuple:

     '''simple docstring'''




     lowerCamelCase__  =     sequences
     lowerCamelCase__  =     char_preds.size(0     )

     lowerCamelCase__  =     self._decode_helper(_A				,     "char"     )
     lowerCamelCase__  =     self._decode_helper(_A				,     "bpe"     )
     lowerCamelCase__  =     self._decode_helper(_A				,     "wp"     )

     lowerCamelCase__  =     []
     lowerCamelCase__  =     []
     for i in range(_A     ):
      lowerCamelCase__  =     [char_scores[i], bpe_scores[i], wp_scores[i]]
      lowerCamelCase__  =     [char_strs[i], bpe_strs[i], wp_strs[i]]
      lowerCamelCase__  =     scores.index(max(_A     )     )
      final_strs.append(strs[max_score_index]     )
      final_scores.append(scores[max_score_index]     )

     lowerCamelCase__  =     {}
     lowerCamelCase__  =     final_strs
     lowerCamelCase__  =     final_scores
     lowerCamelCase__  =     char_strs
     lowerCamelCase__  =     bpe_strs
     lowerCamelCase__  =     wp_strs
     return out

    def    a__				(					self     :     Any				,     __lowerCamelCase     :     Optional[int]				,     __lowerCamelCase     :     List[Any]     )							->							Optional[int]:

     '''simple docstring'''




     if format == DecodeType.CHARACTER:
      lowerCamelCase__  =     self.char_decode
      lowerCamelCase__  =     1
      lowerCamelCase__  =     '[s]'
     elif format == DecodeType.BPE:
      lowerCamelCase__  =     self.bpe_decode
      lowerCamelCase__  =     2
      lowerCamelCase__  =     '#'
     elif format == DecodeType.WORDPIECE:
      lowerCamelCase__  =     self.wp_decode
      lowerCamelCase__  =     102
      lowerCamelCase__  =     '[SEP]'
     else:
      raise ValueError(f'''Format {format} is not supported.'''     )

     lowerCamelCase__  =     [], []
     lowerCamelCase__  =     pred_logits.size(0     )
     lowerCamelCase__  =     pred_logits.size(1     )
     lowerCamelCase__  =     pred_logits.topk(1				,     dim=-1				,     largest=_A				,     sorted=_A     )
     lowerCamelCase__  =     preds_index.view(-1				,     _A     )[:, 1:]
     lowerCamelCase__  =     decoder(_A     )
     lowerCamelCase__  =     torch.nn.functional.softmax(_A				,     dim=2     ).max(dim=2     )
     lowerCamelCase__  =     preds_max_prob[:, 1:]

     for index in range(_A     ):
      lowerCamelCase__  =     preds_str[index].find(_A     )
      lowerCamelCase__  =     preds_str[index][:pred_eos]
      lowerCamelCase__  =     preds_index[index].cpu().tolist()
      lowerCamelCase__  =     pred_index.index(_A     ) if eos_token in pred_index else -1
      lowerCamelCase__  =     preds_max_prob[index][: pred_eos_index + 1]
      lowerCamelCase__  =     pred_max_prob.cumprod(dim=0     )[-1] if pred_max_prob.nelement() != 0 else 0.0
      dec_strs.append(_A     )
      conf_scores.append(_A     )

     return dec_strs, conf_scores

    def    a__				(					self     :     Dict				,     __lowerCamelCase     :     Dict     )							->							Union[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     [seq.replace(" "				,     ""     ) for seq in self.char_tokenizer.batch_decode(_A     )]
     return decode_strs

    def    a__				(					self     :     Any				,     __lowerCamelCase     :     Union[str, Any]     )							->							str:

     '''simple docstring'''




     return self.bpe_tokenizer.batch_decode(_A     )




    def    a__				(					self     :     Union[str, Any]				,     __lowerCamelCase     :     int     )							->							Tuple:

     '''simple docstring'''




     lowerCamelCase__  =     [seq.replace(" "				,     ""     ) for seq in self.wp_tokenizer.batch_decode(_A     )]
     return decode_strs









'''simple docstring'''





# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
 import jax
 import jaxlib

__A							:							Any  							=		get_logger()

__A							:							Optional[dict]  							=		None
class 				lowercase       (							TensorFormatter[Mapping, "jax.Array", Mapping]      ):


    '''simple docstring'''

    def __init__(					self     :     str				,     __lowerCamelCase     :     Optional[Any]=None				,     __lowerCamelCase     :     Dict=None				,     **__lowerCamelCase     :     str     )							->							List[Any]:

     '''simple docstring'''




     super().__init__(features=__lowerCamelCase     )
     import jax
     from jaxlib.xla_client import Device

     if isinstance(__lowerCamelCase				,     __lowerCamelCase     ):
      raise ValueError(
          f'''Expected {device} to be a `str` not {type(__lowerCamelCase     )}, as `jaxlib.xla_extension.Device` '''
          "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
          "the device with `str()` to get its string identifier that will be internally mapped "
          "to the actual `jaxlib.xla_extension.Device`."     )
     lowerCamelCase__  =     device if isinstance(__lowerCamelCase				,     __lowerCamelCase     ) else str(jax.devices()[0]     )
     # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
     # with `pickle` nor with `dill`, so we need to use a global variable instead
     global DEVICE_MAPPING
     if DEVICE_MAPPING is None:
      lowerCamelCase__  =     self._map_devices_to_str()
     if self.device not in list(DEVICE_MAPPING.keys()     ):
      logger.warning(
          f'''Device with string identifier {self.device} not listed among the available '''
          f'''devices: {list(DEVICE_MAPPING.keys()     )}, so falling back to the default '''
          f'''device: {str(jax.devices()[0]     )}.'''     )
      lowerCamelCase__  =     str(jax.devices()[0]     )
     lowerCamelCase__  =     jnp_array_kwargs

    @staticmethod
    def    a__				(					)							->							Dict[str, "jaxlib.xla_extension.Device"]:

     '''simple docstring'''




     import jax

     return {str(__lowerCamelCase     ): device for device in jax.devices()}

    def    a__				(					self     :     Optional[Any]				,     __lowerCamelCase     :     Optional[int]     )							->							List[str]:

     '''simple docstring'''




     import jax
     import jax.numpy as jnp

     if isinstance(__lowerCamelCase				,     __lowerCamelCase     ) and column:
      if all(
          isinstance(__lowerCamelCase				,     jax.Array     ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column     ):
       return jnp.stack(__lowerCamelCase				,     axis=0     )
     return column

    def    a__				(					self     :     List[Any]				,     __lowerCamelCase     :     List[Any]     )							->							Optional[int]:

     '''simple docstring'''




     import jax
     import jax.numpy as jnp

     if isinstance(__lowerCamelCase				,     (str, bytes, type(__lowerCamelCase     ))     ):
      return value
     elif isinstance(__lowerCamelCase				,     (np.character, np.ndarray)     ) and np.issubdtype(value.dtype				,     np.character     ):
      return value.tolist()

     lowerCamelCase__  =     {}

     if isinstance(__lowerCamelCase				,     (np.number, np.ndarray)     ) and np.issubdtype(value.dtype				,     np.integer     ):
      # the default int precision depends on the jax config
      # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
      if jax.config.jax_enable_xaa:
       lowerCamelCase__  =     {"dtype": jnp.intaa}
      else:
       lowerCamelCase__  =     {"dtype": jnp.intaa}
     elif isinstance(__lowerCamelCase				,     (np.number, np.ndarray)     ) and np.issubdtype(value.dtype				,     np.floating     ):
      lowerCamelCase__  =     {"dtype": jnp.floataa}
     elif config.PIL_AVAILABLE and "PIL" in sys.modules:
      import PIL.Image

      if isinstance(__lowerCamelCase				,     PIL.Image.Image     ):
       lowerCamelCase__  =     np.asarray(__lowerCamelCase     )

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
     global DEVICE_MAPPING
     if DEVICE_MAPPING is None:
      lowerCamelCase__  =     self._map_devices_to_str()

     with jax.default_device(DEVICE_MAPPING[self.device]     ):
      # calling jnp.array on a np.ndarray does copy the data
      # see https://github.com/google/jax/issues/4486
      return jnp.array(__lowerCamelCase				,     **{**default_dtype, **self.jnp_array_kwargs}     )

    def    a__				(					self     :     List[str]				,     __lowerCamelCase     :     Optional[Any]     )							->							Dict:

     '''simple docstring'''




     import jax

     # support for torch, tf, jax etc.
     if config.TORCH_AVAILABLE and "torch" in sys.modules:
      import torch

      if isinstance(__lowerCamelCase				,     torch.Tensor     ):
       return self._tensorize(data_struct.detach().cpu().numpy()[()]     )
     if hasattr(__lowerCamelCase				,     "__array__"     ) and not isinstance(__lowerCamelCase				,     jax.Array     ):
      lowerCamelCase__  =     data_struct.__array__()
     # support for nested types like struct of list of struct
     if isinstance(__lowerCamelCase				,     np.ndarray     ):
      if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
       return self._consolidate([self.recursive_tensorize(__lowerCamelCase     ) for substruct in data_struct]     )
     elif isinstance(__lowerCamelCase				,     (list, tuple)     ):
      return self._consolidate([self.recursive_tensorize(__lowerCamelCase     ) for substruct in data_struct]     )
     return self._tensorize(__lowerCamelCase     )

    def    a__				(					self     :     Any				,     __lowerCamelCase     :     dict     )							->							int:

     '''simple docstring'''




     return map_nested(self._recursive_tensorize				,     __lowerCamelCase				,     map_list=__lowerCamelCase     )

    def    a__				(					self     :     Union[str, Any]				,     __lowerCamelCase     :     pa.Table     )							->							Mapping:

     '''simple docstring'''




     lowerCamelCase__  =     self.numpy_arrow_extractor().extract_row(__lowerCamelCase     )
     lowerCamelCase__  =     self.python_features_decoder.decode_row(__lowerCamelCase     )
     return self.recursive_tensorize(__lowerCamelCase     )

    def    a__				(					self     :     List[Any]				,     __lowerCamelCase     :     pa.Table     )							->							"jax.Array":

     '''simple docstring'''




     lowerCamelCase__  =     self.numpy_arrow_extractor().extract_column(__lowerCamelCase     )
     lowerCamelCase__  =     self.python_features_decoder.decode_column(__lowerCamelCase				,     pa_table.column_names[0]     )
     lowerCamelCase__  =     self.recursive_tensorize(__lowerCamelCase     )
     lowerCamelCase__  =     self._consolidate(__lowerCamelCase     )
     return column




    def    a__				(					self     :     List[str]				,     __lowerCamelCase     :     pa.Table     )							->							Mapping:

     '''simple docstring'''




     lowerCamelCase__  =     self.numpy_arrow_extractor().extract_batch(__lowerCamelCase     )
     lowerCamelCase__  =     self.python_features_decoder.decode_batch(__lowerCamelCase     )
     lowerCamelCase__  =     self.recursive_tensorize(__lowerCamelCase     )
     for column_name in batch:
      lowerCamelCase__  =     self._consolidate(batch[column_name]     )
     return batch

