




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

lowercase__							:	str											=		get_logger()

lowercase__							:	Optional[dict]											=		None
class 						UpperCAmelCase  (							TensorFormatter[Mapping, '''jax.Array''', Mapping]     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[Any]			,      __lowercase				:	Optional[int]=None			,      __lowercase				:	Optional[int]=None			,      **__lowercase				:	Tuple		):






							"""simple docstring"""






							super().__init__(features=__lowercase		)
							import jax
							from jaxlib.xla_client import Device

							if isinstance(__lowercase			,      __lowercase		):
								raise ValueError(
								    f"Expected {device} to be a `str` not {type(__lowercase		)}, as `jaxlib.xla_extension.Device` "
								    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
								    "the device with `str()` to get its string identifier that will be internally mapped "
								    "to the actual `jaxlib.xla_extension.Device`."		)
							snake_case_						       =	device if isinstance(__lowercase			,      __lowercase		) else str(jax.devices()[0]		)
							# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
							# with `pickle` nor with `dill`, so we need to use a global variable instead
							global DEVICE_MAPPING
							if DEVICE_MAPPING is None:
								snake_case_						       =	self._map_devices_to_str()
							if self.device not in list(DEVICE_MAPPING.keys()		):
								logger.warning(
								    f"Device with string identifier {self.device} not listed among the available "
								    f"devices: {list(DEVICE_MAPPING.keys()		)}, so falling back to the default "
								    f"device: {str(jax.devices()[0]		)}."		)
								snake_case_						       =	str(jax.devices()[0]		)
							snake_case_						       =	jnp_array_kwargs






						@staticmethod
						def 			snake_case__		(		):






							"""simple docstring"""






							import jax

							return {str(__lowercase		): device for device in jax.devices()}






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							import jax
							import jax.numpy as jnp

							if isinstance(__lowercase			,      __lowercase		) and column:
								if all(
								    isinstance(__lowercase			,      jax.Array		) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column		):
									return jnp.stack(__lowercase			,      axis=0		)
							return column






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							import jax
							import jax.numpy as jnp

							if isinstance(__lowercase			,      (str, bytes, type(__lowercase		))		):
								return value
							elif isinstance(__lowercase			,      (np.character, np.ndarray)		) and np.issubdtype(value.dtype			,      np.character		):
								return value.tolist()

							snake_case_						       =	{}

							if isinstance(__lowercase			,      (np.number, np.ndarray)		) and np.issubdtype(value.dtype			,      np.integer		):
								# the default int precision depends on the jax config
								# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
								if jax.config.jax_enable_xaa:
									snake_case_						       =	{"dtype": jnp.intaa}
								else:
									snake_case_						       =	{"dtype": jnp.intaa}
							elif isinstance(__lowercase			,      (np.number, np.ndarray)		) and np.issubdtype(value.dtype			,      np.floating		):
								snake_case_						       =	{"dtype": jnp.floataa}
							elif config.PIL_AVAILABLE and "PIL" in sys.modules:
								import PIL.Image

								if isinstance(__lowercase			,      PIL.Image.Image		):
									snake_case_						       =	np.asarray(__lowercase		)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
							global DEVICE_MAPPING
							if DEVICE_MAPPING is None:
								snake_case_						       =	self._map_devices_to_str()

							with jax.default_device(DEVICE_MAPPING[self.device]		):
								# calling jnp.array on a np.ndarray does copy the data
								# see https://github.com/google/jax/issues/4486
								return jnp.array(__lowercase			,      **{**default_dtype, **self.jnp_array_kwargs}		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							import jax

							# support for torch, tf, jax etc.
							if config.TORCH_AVAILABLE and "torch" in sys.modules:
								import torch

								if isinstance(__lowercase			,      torch.Tensor		):
									return self._tensorize(data_struct.detach().cpu().numpy()[()]		)
							if hasattr(__lowercase			,      "__array__"		) and not isinstance(__lowercase			,      jax.Array		):
								snake_case_						       =	data_struct.__array__()
							# support for nested types like struct of list of struct
							if isinstance(__lowercase			,      np.ndarray		):
								if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
									return self._consolidate([self.recursive_tensorize(__lowercase		) for substruct in data_struct]		)
							elif isinstance(__lowercase			,      (list, tuple)		):
								return self._consolidate([self.recursive_tensorize(__lowercase		) for substruct in data_struct]		)
							return self._tensorize(__lowercase		)






						def 			snake_case__		(		self				:	str			,      __lowercase				:	dict		):






							"""simple docstring"""






							return map_nested(self._recursive_tensorize			,      __lowercase			,      map_list=__lowercase		)






						def 			snake_case__		(		self				:	Tuple			,      __lowercase				:	pa.Table		):






							"""simple docstring"""






							snake_case_						       =	self.numpy_arrow_extractor().extract_row(__lowercase		)
							snake_case_						       =	self.python_features_decoder.decode_row(__lowercase		)
							return self.recursive_tensorize(__lowercase		)






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	pa.Table		):






							"""simple docstring"""






							snake_case_						       =	self.numpy_arrow_extractor().extract_column(__lowercase		)
							snake_case_						       =	self.python_features_decoder.decode_column(__lowercase			,      pa_table.column_names[0]		)
							snake_case_						       =	self.recursive_tensorize(__lowercase		)
							snake_case_						       =	self._consolidate(__lowercase		)
							return column






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	pa.Table		):






							"""simple docstring"""






							snake_case_						       =	self.numpy_arrow_extractor().extract_batch(__lowercase		)
							snake_case_						       =	self.python_features_decoder.decode_batch(__lowercase		)
							snake_case_						       =	self.recursive_tensorize(__lowercase		)
							for column_name in batch:
								snake_case_						       =	self._consolidate(batch[column_name]		)
							return batch






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class 						UpperCAmelCase  (							UpperCAmelCase__			,  UpperCAmelCase__			,  UpperCAmelCase__     ):







						'''simple docstring'''





						@register_to_config
						def __init__(		self				:	Tuple			,      __lowercase				:	int			,      __lowercase				:	int			,      __lowercase				:	int			,      __lowercase				:	float			,      __lowercase				:	int			,      __lowercase				:	int			,      __lowercase				:	int			,      __lowercase				:	int			,      __lowercase				:	str			,      __lowercase				:	bool = False			,      ):






							"""simple docstring"""






							super().__init__()

							snake_case_						       =	nn.Embedding(__lowercase			,      __lowercase		)

							snake_case_						       =	nn.Embedding(__lowercase			,      __lowercase		)
							snake_case_						       =	False

							snake_case_						       =	nn.Dropout(p=__lowercase		)

							snake_case_						       =	TaConfig(
							    vocab_size=__lowercase			,      d_model=__lowercase			,      num_heads=__lowercase			,      d_kv=__lowercase			,      d_ff=__lowercase			,      dropout_rate=__lowercase			,      feed_forward_proj=__lowercase			,      is_decoder=__lowercase			,      is_encoder_decoder=__lowercase			,      )

							snake_case_						       =	nn.ModuleList()
							for lyr_num in range(__lowercase		):
								snake_case_						       =	TaBlock(__lowercase		)
								self.encoders.append(__lowercase		)

							snake_case_						       =	TaLayerNorm(__lowercase		)
							snake_case_						       =	nn.Dropout(p=__lowercase		)






						def 			snake_case__		(		self				:	str			,      __lowercase				:	Dict			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	self.token_embedder(__lowercase		)

							snake_case_						       =	encoder_input_tokens.shape[1]
							snake_case_						       =	torch.arange(__lowercase			,      device=encoder_input_tokens.device		)
							x += self.position_encoding(__lowercase		)

							snake_case_						       =	self.dropout_pre(__lowercase		)

							# inverted the attention mask
							snake_case_						       =	encoder_input_tokens.size()
							snake_case_						       =	self.get_extended_attention_mask(__lowercase			,      __lowercase		)

							for lyr in self.encoders:
								snake_case_						       =	lyr(__lowercase			,      __lowercase		)[0]
							snake_case_						       =	self.layer_norm(__lowercase		)

							return self.dropout_post(__lowercase		), encoder_inputs_mask
