



import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case    :						List[str]  =						get_tests_dir('fixtures/test_sentencepiece.model')

if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

snake_case    :						str  =						2_5_0_0_0_4
snake_case    :						Union[str, Any]  =						2_5_0_0_2_0




@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__  (	UpperCamelCase					,							unittest.TestCase						):
					__A     :    Any           =						MBartaaTokenizer
					__A     :    Dict           =						MBartaaTokenizerFast
					__A     :    Tuple           =						True
					__A     :    Optional[Any]           =						True







					def  _lowercase			(       self      :     Union[str, Any]):
											super().setUp()

											# We have a SentencePiece fixture for testing
											A__	:							Union[str, Any]						    = MBartaaTokenizer(_A			,		src_lang="en_XX"			,		tgt_lang="ro_RO"			,		keep_accents=_A)
											tokenizer.save_pretrained(self.tmpdirname)







					def  _lowercase			(       self      :     Dict):
											A__	:							Union[str, Any]						    = "<s>"
											A__	:							Union[str, Any]						    = 0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)			,		_A)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)			,		_A)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Union[str, Any]						    = list(self.get_tokenizer().get_vocab().keys())

											self.assertEqual(vocab_keys[0]			,		"<s>")
											self.assertEqual(vocab_keys[1]			,		"<pad>")
											self.assertEqual(vocab_keys[-1]			,		"<mask>")
											self.assertEqual(len(_A)			,		1054)







					def  _lowercase			(       self      :     int):
											self.assertEqual(self.get_tokenizer().vocab_size			,		1054)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Tuple						    = MBartaaTokenizer(_A			,		src_lang="en_XX"			,		tgt_lang="ro_RO"			,		keep_accents=_A)

											A__	:							List[str]						    = tokenizer.tokenize("This is a test")
											self.assertListEqual(_A			,		["▁This", "▁is", "▁a", "▁t", "est"])

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(_A)			,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,		)

											A__	:							List[str]						    = tokenizer.tokenize("I was born in 92000, and this is falsé.")
											self.assertListEqual(
											    _A			,		[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]			,		)
											A__	:							Optional[Any]						    = tokenizer.convert_tokens_to_ids(_A)
											self.assertListEqual(
											    _A			,		[
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
											    ]			,		)

											A__	:							int						    = tokenizer.convert_ids_to_tokens(_A)
											self.assertListEqual(
											    _A			,		[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]			,		)







					@slow
					def  _lowercase			(       self      :     List[Any]):
											# fmt: off
											A__	:							List[str]						    = {"input_ids": [[25_0004, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [25_0004, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_0004, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A			,		model_name="facebook/mbart-large-50"			,		revision="d3913889c59cd5c9e456b269c376325eabad57e2"			,		)







					def  _lowercase			(       self      :     Any):
											if not self.test_slow_tokenizer:
																	# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																	return

											A__	:							Optional[Any]						    = (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-mbart50", {})
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																							A__	:							List[Any]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)
																							A__	:							Tuple						    = self.tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							Optional[Any]						    = tempfile.mkdtemp()

																							A__	:							str						    = tokenizer_r.save_pretrained(_A)
																							A__	:							Union[str, Any]						    = tokenizer_p.save_pretrained(_A)

																							# Checks it save with the same files + the tokenizer.json file for the fast one
																							self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))
																							A__	:							Dict						    = tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f)
																							self.assertSequenceEqual(_A			,		_A)

																							# Checks everything loads correctly in the same way
																							A__	:							Union[str, Any]						    = tokenizer_r.from_pretrained(_A)
																							A__	:							List[str]						    = tokenizer_p.from_pretrained(_A)

																							# Check special tokens are set accordingly on Rust and Python
																							for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(_A			,		_A))
																													# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																													# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																							shutil.rmtree(_A)

																							# Save tokenizer rust, legacy_format=True
																							A__	:							List[Any]						    = tempfile.mkdtemp()

																							A__	:							Union[str, Any]						    = tokenizer_r.save_pretrained(_A			,		legacy_format=_A)
																							A__	:							Any						    = tokenizer_p.save_pretrained(_A)

																							# Checks it save with the same files
																							self.assertSequenceEqual(_A			,		_A)

																							# Checks everything loads correctly in the same way
																							A__	:							int						    = tokenizer_r.from_pretrained(_A)
																							A__	:							Dict						    = tokenizer_p.from_pretrained(_A)

																							# Check special tokens are set accordingly on Rust and Python
																							for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(_A			,		_A))

																							shutil.rmtree(_A)

																							# Save tokenizer rust, legacy_format=False
																							A__	:							List[Any]						    = tempfile.mkdtemp()

																							A__	:							str						    = tokenizer_r.save_pretrained(_A			,		legacy_format=_A)
																							A__	:							List[str]						    = tokenizer_p.save_pretrained(_A)

																							# Checks it saved the tokenizer.json file
																							self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))

																							# Checks everything loads correctly in the same way
																							A__	:							List[Any]						    = tokenizer_r.from_pretrained(_A)
																							A__	:							List[Any]						    = tokenizer_p.from_pretrained(_A)

																							# Check special tokens are set accordingly on Rust and Python
																							for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(_A			,		_A))

																							shutil.rmtree(_A)




@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__  (	unittest.TestCase						):
					__A     :    List[str]           =						'facebook/mbart-large-50-one-to-many-mmt'
					__A     :    Any           =						[
					    ' UN Chief Says There Is No Military Solution in Syria',
					    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
					]
					__A     :    List[Any]           =						[
					    'Şeful ONU declară că nu există o soluţie militară în Siria',
					    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
					    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
					    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
					]
					__A     :    int           =						[EN_CODE, 82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2]







					@classmethod
					def  _lowercase			(       cls      :     Optional[int]):
											A__	:							MBartaaTokenizer						    = MBartaaTokenizer.from_pretrained(
											    cls.checkpoint_name			,		src_lang="en_XX"			,		tgt_lang="ro_RO")
											A__	:							Tuple						    = 1
											return cls







					def  _lowercase			(       self      :     str):
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ar_AR"]			,		25_0001)
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["en_EN"]			,		25_0004)
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ro_RO"]			,		25_0020)
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["mr_IN"]			,		25_0038)







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							Tuple						    = self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
											self.assertListEqual(self.expected_src_tokens			,		_A)







					def  _lowercase			(       self      :     Optional[Any]):
											self.assertIn(_A			,		self.tokenizer.all_special_ids)
											A__	:							Dict						    = [RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
											A__	:							Tuple						    = self.tokenizer.decode(_A			,		skip_special_tokens=_A)
											A__	:							Any						    = self.tokenizer.decode(generated_ids[1:]			,		skip_special_tokens=_A)
											self.assertEqual(_A			,		_A)
											self.assertNotIn(self.tokenizer.eos_token			,		_A)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							List[str]						    = ["this is gunna be a long sentence " * 20]
											assert isinstance(src_text[0]			,		_A)
											A__	:							str						    = 10
											A__	:							Optional[int]						    = self.tokenizer(_A			,		max_length=_A			,		truncation=_A).input_ids[0]
											self.assertEqual(ids[0]			,		_A)
											self.assertEqual(ids[-1]			,		2)
											self.assertEqual(len(_A)			,		_A)







					def  _lowercase			(       self      :     str):
											self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"])			,		[25_0053, 25_0001])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Any						    = tempfile.mkdtemp()
											A__	:							Tuple						    = self.tokenizer.fairseq_tokens_to_ids
											self.tokenizer.save_pretrained(_A)
											A__	:							Dict						    = MBartaaTokenizer.from_pretrained(_A)
											self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,		_A)







					@require_torch
					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							List[Any]						    = self.tokenizer(self.src_text			,		text_target=self.tgt_text			,		padding=_A			,		return_tensors="pt")
											A__	:							Any						    = shift_tokens_right(batch["labels"]			,		self.tokenizer.pad_token_id)

											# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
											assert batch.input_ids[1][0] == EN_CODE
											assert batch.input_ids[1][-1] == 2
											assert batch.labels[1][0] == RO_CODE
											assert batch.labels[1][-1] == 2
											assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]







					@require_torch
					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							List[Any]						    = self.tokenizer(
											    self.src_text			,		text_target=self.tgt_text			,		padding=_A			,		truncation=_A			,		max_length=len(self.expected_src_tokens)			,		return_tensors="pt"			,		)
											A__	:							Optional[int]						    = shift_tokens_right(batch["labels"]			,		self.tokenizer.pad_token_id)

											self.assertIsInstance(_A			,		_A)

											self.assertEqual((2, 14)			,		batch.input_ids.shape)
											self.assertEqual((2, 14)			,		batch.attention_mask.shape)
											A__	:							List[Any]						    = batch.input_ids.tolist()[0]
											self.assertListEqual(self.expected_src_tokens			,		_A)
											self.assertEqual(2			,		batch.decoder_input_ids[0, 0])  # decoder_start_token_id
											# Test that special tokens are reset
											self.assertEqual(self.tokenizer.prefix_tokens			,		[EN_CODE])
											self.assertEqual(self.tokenizer.suffix_tokens			,		[self.tokenizer.eos_token_id])







					def  _lowercase			(       self      :     List[str]):
											A__	:							Optional[int]						    = self.tokenizer(self.src_text			,		padding=_A			,		truncation=_A			,		max_length=3			,		return_tensors="pt")
											A__	:							Optional[Any]						    = self.tokenizer(
											    text_target=self.tgt_text			,		padding=_A			,		truncation=_A			,		max_length=10			,		return_tensors="pt")
											A__	:							str						    = targets["input_ids"]
											A__	:							List[str]						    = shift_tokens_right(_A			,		self.tokenizer.pad_token_id)

											self.assertEqual(batch.input_ids.shape[1]			,		3)
											self.assertEqual(batch.decoder_input_ids.shape[1]			,		10)







					@require_torch
					def  _lowercase			(       self      :     int):
											A__	:							Any						    = self.tokenizer._build_translation_inputs(
											    "A test"			,		return_tensors="pt"			,		src_lang="en_XX"			,		tgt_lang="ar_AR")

											self.assertEqual(
											    nested_simplify(_A)			,		{
											        # en_XX, A, test, EOS
											        "input_ids": [[25_0004, 62, 3034, 2]],
											        "attention_mask": [[1, 1, 1, 1]],
											        # ar_AR
											        "forced_bos_token_id": 25_0001,
											    }			,		)




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

snake_case    :						Optional[int]  =						get_logger()

snake_case    :						Optional[dict]  =						None




class 							lowerCAmelCase__  (	TensorFormatter[Mapping, 'jax.Array', Mapping]						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Optional[int]=None			,		_A      :     str=None			,		**_A      :     Tuple):
											super().__init__(features=_A)
											import jax
											from jaxlib.xla_client import Device

											if isinstance(_A			,		_A):
																	raise ValueError(
																	    F'Expected {device} to be a `str` not {type(_A)}, as `jaxlib.xla_extension.Device` '
																	    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
																	    "the device with `str()` to get its string identifier that will be internally mapped "
																	    "to the actual `jaxlib.xla_extension.Device`.")
											A__	:							Union[str, Any]						    = device if isinstance(_A			,		_A) else str(jax.devices()[0])
											# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
											# with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Any						    = self._map_devices_to_str()
											if self.device not in list(DEVICE_MAPPING.keys()):
																	logger.warning(
																	    F'Device with string identifier {self.device} not listed among the available '
																	    F'devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default '
																	    F'device: {str(jax.devices()[0])}.')
																	A__	:							str						    = str(jax.devices()[0])
											A__	:							List[Any]						    = jnp_array_kwargs







					@staticmethod
					def  _lowercase			(       ):
											import jax

											return {str(_A): device for device in jax.devices()}







					def  _lowercase			(       self      :     Tuple			,		_A      :     int):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		_A) and column:
																	if all(
																	    isinstance(_A			,		jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column):
																							return jnp.stack(_A			,		axis=0)
											return column







					def  _lowercase			(       self      :     List[str]			,		_A      :     Union[str, Any]):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		(str, bytes, type(_A))):
																	return value
											elif isinstance(_A			,		(np.character, np.ndarray)) and np.issubdtype(value.dtype			,		np.character):
																	return value.tolist()

											A__	:							Dict						    = {}

											if isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.integer):
																	# the default int precision depends on the jax config
																	# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
																	if jax.config.jax_enable_xaa:
																							A__	:							Union[str, Any]						    = {"dtype": jnp.intaa}
																	else:
																							A__	:							int						    = {"dtype": jnp.intaa}
											elif isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.floating):
																	A__	:							str						    = {"dtype": jnp.floataa}
											elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																	import PIL.Image

																	if isinstance(_A			,		PIL.Image.Image):
																							A__	:							Any						    = np.asarray(_A)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Optional[Any]						    = self._map_devices_to_str()

											with jax.default_device(DEVICE_MAPPING[self.device]):
																	# calling jnp.array on a np.ndarray does copy the data
																	# see https://github.com/google/jax/issues/4486
																	return jnp.array(_A			,		**{**default_dtype, **self.jnp_array_kwargs})







					def  _lowercase			(       self      :     Dict			,		_A      :     Any):
											import jax

											# support for torch, tf, jax etc.
											if config.TORCH_AVAILABLE and "torch" in sys.modules:
																	import torch

																	if isinstance(_A			,		torch.Tensor):
																							return self._tensorize(data_struct.detach().cpu().numpy()[()])
											if hasattr(_A			,		"__array__") and not isinstance(_A			,		jax.Array):
																	A__	:							Any						    = data_struct.__array__()
											# support for nested types like struct of list of struct
											if isinstance(_A			,		np.ndarray):
																	if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																							return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											elif isinstance(_A			,		(list, tuple)):
																	return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											return self._tensorize(_A)







					def  _lowercase			(       self      :     Optional[Any]			,		_A      :     dict):
											return map_nested(self._recursive_tensorize			,		_A			,		map_list=_A)







					def  _lowercase			(       self      :     List[str]			,		_A      :     pa.Table):
											A__	:							Dict						    = self.numpy_arrow_extractor().extract_row(_A)
											A__	:							int						    = self.python_features_decoder.decode_row(_A)
											return self.recursive_tensorize(_A)







					def  _lowercase			(       self      :     int			,		_A      :     pa.Table):
											A__	:							int						    = self.numpy_arrow_extractor().extract_column(_A)
											A__	:							Optional[int]						    = self.python_features_decoder.decode_column(_A			,		pa_table.column_names[0])
											A__	:							Dict						    = self.recursive_tensorize(_A)
											A__	:							List[Any]						    = self._consolidate(_A)
											return column







					def  _lowercase			(       self      :     Tuple			,		_A      :     pa.Table):
											A__	:							Any						    = self.numpy_arrow_extractor().extract_batch(_A)
											A__	:							str						    = self.python_features_decoder.decode_batch(_A)
											A__	:							Optional[Any]						    = self.recursive_tensorize(_A)
											for column_name in batch:
																	A__	:							Optional[int]						    = self._consolidate(batch[column_name])
											return batch