



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 							lowerCAmelCase__  (	UpperCamelCase					,							unittest.TestCase						):
					__A     :    Optional[int]           =						BertTokenizer
					__A     :    str           =						BertTokenizerFast
					__A     :    Optional[Any]           =						True
					__A     :    Union[str, Any]           =						True
					__A     :    Union[str, Any]           =						filter_non_english







					def  _lowercase			(       self      :     str):
											super().setUp()

											A__	:							List[Any]						    = [
											    "[UNK]",
											    "[CLS]",
											    "[SEP]",
											    "[PAD]",
											    "[MASK]",
											    "want",
											    "##want",
											    "##ed",
											    "wa",
											    "un",
											    "runn",
											    "##ing",
											    ",",
											    "low",
											    "lowest",
											]
											A__	:							Any						    = os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES["vocab_file"])
											with open(self.vocab_file			,		"w"			,		encoding="utf-8") as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))







					def  _lowercase			(       self      :     Any			,		_A      :     int):
											A__	:							List[str]						    = "UNwant\u00E9d,running"
											A__	:							List[Any]						    = "unwanted, running"
											return input_text, output_text







					def  _lowercase			(       self      :     Any):
											A__	:							Tuple						    = self.tokenizer_class(self.vocab_file)

											A__	:							str						    = tokenizer.tokenize("UNwant\u00E9d,running")
											self.assertListEqual(_A			,		["un", "##want", "##ed", ",", "runn", "##ing"])
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)			,		[9, 6, 7, 12, 10, 11])







					def  _lowercase			(       self      :     Dict):
											if not self.test_rust_tokenizer:
																	return

											A__	:							Union[str, Any]						    = self.get_tokenizer()
											A__	:							List[str]						    = self.get_rust_tokenizer()

											A__	:							List[Any]						    = "UNwant\u00E9d,running"

											A__	:							Union[str, Any]						    = tokenizer.tokenize(_A)
											A__	:							List[str]						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							Tuple						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							str						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = self.get_rust_tokenizer()
											A__	:							int						    = tokenizer.encode(_A)
											A__	:							Tuple						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)

											# With lower casing
											A__	:							Any						    = self.get_tokenizer(do_lower_case=_A)
											A__	:							List[str]						    = self.get_rust_tokenizer(do_lower_case=_A)

											A__	:							str						    = "UNwant\u00E9d,running"

											A__	:							str						    = tokenizer.tokenize(_A)
											A__	:							List[str]						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							Optional[int]						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = self.get_rust_tokenizer()
											A__	:							int						    = tokenizer.encode(_A)
											A__	:							List[str]						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)







					def  _lowercase			(       self      :     Dict):
											A__	:							Union[str, Any]						    = BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")			,		["ah", "\u535A", "\u63A8", "zz"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							str						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,		["hello", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     List[str]):
											A__	:							Union[str, Any]						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hällo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["h\u00E9llo"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							Union[str, Any]						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hallo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     Dict):
											A__	:							List[Any]						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hallo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     Dict):
											A__	:							Optional[int]						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,		["HeLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Tuple):
											A__	:							Tuple						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["HäLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Dict						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["HaLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							List[str]						    = BasicTokenizer(do_lower_case=_A			,		never_split=["[UNK]"])

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")			,		["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])







					def  _lowercase			(       self      :     List[Any]):
											A__	:							List[Any]						    = BasicTokenizer()
											A__	:							Optional[int]						    = "a\n'll !!to?'d of, can't."
											A__	:							Tuple						    = ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
											self.assertListEqual(tokenizer.tokenize(_A)			,		_A)







					def  _lowercase			(       self      :     Tuple):
											A__	:							List[str]						    = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

											A__	:							Dict						    = {}
											for i, token in enumerate(_A):
																	A__	:							Tuple						    = i
											A__	:							Dict						    = WordpieceTokenizer(vocab=_A			,		unk_token="[UNK]")

											self.assertListEqual(tokenizer.tokenize("")			,		[])

											self.assertListEqual(tokenizer.tokenize("unwanted running")			,		["un", "##want", "##ed", "runn", "##ing"])

											self.assertListEqual(tokenizer.tokenize("unwantedX running")			,		["[UNK]", "runn", "##ing"])







					def  _lowercase			(       self      :     Dict):
											self.assertTrue(_is_whitespace(" "))
											self.assertTrue(_is_whitespace("\t"))
											self.assertTrue(_is_whitespace("\r"))
											self.assertTrue(_is_whitespace("\n"))
											self.assertTrue(_is_whitespace("\u00A0"))

											self.assertFalse(_is_whitespace("A"))
											self.assertFalse(_is_whitespace("-"))







					def  _lowercase			(       self      :     List[str]):
											self.assertTrue(_is_control("\u0005"))

											self.assertFalse(_is_control("A"))
											self.assertFalse(_is_control(" "))
											self.assertFalse(_is_control("\t"))
											self.assertFalse(_is_control("\r"))







					def  _lowercase			(       self      :     str):
											self.assertTrue(_is_punctuation("-"))
											self.assertTrue(_is_punctuation("$"))
											self.assertTrue(_is_punctuation("`"))
											self.assertTrue(_is_punctuation("."))

											self.assertFalse(_is_punctuation("A"))
											self.assertFalse(_is_punctuation(" "))







					def  _lowercase			(       self      :     Dict):
											A__	:							Optional[Any]						    = self.get_tokenizer()
											A__	:							Dict						    = self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_A) for t in ["Test", "\xad", "test"]]			,		[["[UNK]"], [], ["[UNK]"]])

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_A) for t in ["Test", "\xad", "test"]]			,		[["[UNK]"], [], ["[UNK]"]])







					@slow
					def  _lowercase			(       self      :     List[str]):
											A__	:							int						    = self.tokenizer_class.from_pretrained("bert-base-uncased")

											A__	:							str						    = tokenizer.encode("sequence builders"			,		add_special_tokens=_A)
											A__	:							Dict						    = tokenizer.encode("multi-sequence build"			,		add_special_tokens=_A)

											A__	:							Optional[Any]						    = tokenizer.build_inputs_with_special_tokens(_A)
											A__	:							Union[str, Any]						    = tokenizer.build_inputs_with_special_tokens(_A			,		_A)

											assert encoded_sentence == [101] + text + [102]
											assert encoded_pair == [101] + text + [102] + text_a + [102]







					def  _lowercase			(       self      :     int):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																							A__	:							Union[str, Any]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							List[str]						    = F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																							A__	:							Optional[int]						    = tokenizer_r.encode_plus(
																							    _A			,		return_attention_mask=_A			,		return_token_type_ids=_A			,		return_offsets_mapping=_A			,		add_special_tokens=_A			,		)

																							A__	:							Tuple						    = tokenizer_r.do_lower_case if hasattr(_A			,		"do_lower_case") else False
																							A__	:							Any						    = (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), "A"),
																							        ((1, 2), ","),
																							        ((3, 5), "na"),
																							        ((5, 6), "##ï"),
																							        ((6, 8), "##ve"),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), "Allen"),
																							        ((21, 23), "##NL"),
																							        ((23, 24), "##P"),
																							        ((25, 33), "sentence"),
																							        ((33, 34), "."),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), "a"),
																							        ((1, 2), ","),
																							        ((3, 8), "naive"),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), "allen"),
																							        ((21, 23), "##nl"),
																							        ((23, 24), "##p"),
																							        ((25, 33), "sentence"),
																							        ((33, 34), "."),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]			,		tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
																							self.assertEqual([e[0] for e in expected_results]			,		tokens["offset_mapping"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							str						    = ["的", "人", "有"]
											A__	:							List[Any]						    = "".join(_A)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																							A__	:							str						    = True
																							A__	:							Optional[Any]						    = self.tokenizer_class.from_pretrained(_A			,		**_A)
																							A__	:							List[str]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							List[Any]						    = tokenizer_p.encode(_A			,		add_special_tokens=_A)
																							A__	:							str						    = tokenizer_r.encode(_A			,		add_special_tokens=_A)

																							A__	:							Tuple						    = tokenizer_r.convert_ids_to_tokens(_A)
																							A__	:							List[str]						    = tokenizer_p.convert_ids_to_tokens(_A)

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(_A			,		_A)
																							self.assertListEqual(_A			,		_A)

																							A__	:							Optional[Any]						    = False
																							A__	:							List[Any]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)
																							A__	:							Dict						    = self.tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							Optional[int]						    = tokenizer_r.encode(_A			,		add_special_tokens=_A)
																							A__	:							Any						    = tokenizer_p.encode(_A			,		add_special_tokens=_A)

																							A__	:							Union[str, Any]						    = tokenizer_r.convert_ids_to_tokens(_A)
																							A__	:							Union[str, Any]						    = tokenizer_p.convert_ids_to_tokens(_A)

																							# it is expected that only the first Chinese character is not preceded by "##".
																							A__	:							str						    = [
																							    F'##{token}' if idx != 0 else token for idx, token in enumerate(_A)
																							]
																							self.assertListEqual(_A			,		_A)
																							self.assertListEqual(_A			,		_A)




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

snake_case    :						Optional[int]  =						get_logger()

snake_case    :						Optional[dict]  =						None




class 							lowerCAmelCase__  (	TensorFormatter[Mapping, 'jax.Array', Mapping]						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Optional[int]=None			,		_A      :     str=None			,		**_A      :     Tuple):
											super().__init__(features=_A)
											import jax
											from jaxlib.xla_client import Device

											if isinstance(_A			,		_A):
																	raise ValueError(
																	    F'Expected {device} to be a `str` not {type(_A)}, as `jaxlib.xla_extension.Device` '
																	    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
																	    "the device with `str()` to get its string identifier that will be internally mapped "
																	    "to the actual `jaxlib.xla_extension.Device`.")
											A__	:							Union[str, Any]						    = device if isinstance(_A			,		_A) else str(jax.devices()[0])
											# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
											# with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Any						    = self._map_devices_to_str()
											if self.device not in list(DEVICE_MAPPING.keys()):
																	logger.warning(
																	    F'Device with string identifier {self.device} not listed among the available '
																	    F'devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default '
																	    F'device: {str(jax.devices()[0])}.')
																	A__	:							str						    = str(jax.devices()[0])
											A__	:							List[Any]						    = jnp_array_kwargs







					@staticmethod
					def  _lowercase			(       ):
											import jax

											return {str(_A): device for device in jax.devices()}







					def  _lowercase			(       self      :     Tuple			,		_A      :     int):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		_A) and column:
																	if all(
																	    isinstance(_A			,		jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column):
																							return jnp.stack(_A			,		axis=0)
											return column







					def  _lowercase			(       self      :     List[str]			,		_A      :     Union[str, Any]):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		(str, bytes, type(_A))):
																	return value
											elif isinstance(_A			,		(np.character, np.ndarray)) and np.issubdtype(value.dtype			,		np.character):
																	return value.tolist()

											A__	:							Dict						    = {}

											if isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.integer):
																	# the default int precision depends on the jax config
																	# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
																	if jax.config.jax_enable_xaa:
																							A__	:							Union[str, Any]						    = {"dtype": jnp.intaa}
																	else:
																							A__	:							int						    = {"dtype": jnp.intaa}
											elif isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.floating):
																	A__	:							str						    = {"dtype": jnp.floataa}
											elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																	import PIL.Image

																	if isinstance(_A			,		PIL.Image.Image):
																							A__	:							Any						    = np.asarray(_A)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Optional[Any]						    = self._map_devices_to_str()

											with jax.default_device(DEVICE_MAPPING[self.device]):
																	# calling jnp.array on a np.ndarray does copy the data
																	# see https://github.com/google/jax/issues/4486
																	return jnp.array(_A			,		**{**default_dtype, **self.jnp_array_kwargs})







					def  _lowercase			(       self      :     Dict			,		_A      :     Any):
											import jax

											# support for torch, tf, jax etc.
											if config.TORCH_AVAILABLE and "torch" in sys.modules:
																	import torch

																	if isinstance(_A			,		torch.Tensor):
																							return self._tensorize(data_struct.detach().cpu().numpy()[()])
											if hasattr(_A			,		"__array__") and not isinstance(_A			,		jax.Array):
																	A__	:							Any						    = data_struct.__array__()
											# support for nested types like struct of list of struct
											if isinstance(_A			,		np.ndarray):
																	if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																							return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											elif isinstance(_A			,		(list, tuple)):
																	return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											return self._tensorize(_A)







					def  _lowercase			(       self      :     Optional[Any]			,		_A      :     dict):
											return map_nested(self._recursive_tensorize			,		_A			,		map_list=_A)







					def  _lowercase			(       self      :     List[str]			,		_A      :     pa.Table):
											A__	:							Dict						    = self.numpy_arrow_extractor().extract_row(_A)
											A__	:							int						    = self.python_features_decoder.decode_row(_A)
											return self.recursive_tensorize(_A)







					def  _lowercase			(       self      :     int			,		_A      :     pa.Table):
											A__	:							int						    = self.numpy_arrow_extractor().extract_column(_A)
											A__	:							Optional[int]						    = self.python_features_decoder.decode_column(_A			,		pa_table.column_names[0])
											A__	:							Dict						    = self.recursive_tensorize(_A)
											A__	:							List[Any]						    = self._consolidate(_A)
											return column







					def  _lowercase			(       self      :     Tuple			,		_A      :     pa.Table):
											A__	:							Any						    = self.numpy_arrow_extractor().extract_batch(_A)
											A__	:							str						    = self.python_features_decoder.decode_batch(_A)
											A__	:							Optional[Any]						    = self.recursive_tensorize(_A)
											for column_name in batch:
																	A__	:							Optional[int]						    = self._consolidate(batch[column_name])
											return batch