

import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch

def 		UpperCamelCase   (lowercase_:					int   )	-> Any:  # picklable for multiprocessing
	return x.sum()

def 		UpperCamelCase   (lowercase_:					Tuple   )	-> str:  # picklable for multiprocessing
	return i + 1



@dataclass
class        _a		:


					'''simple docstring'''






					UpperCAmelCase__:  int
					UpperCAmelCase__:  str



class        _a		(__magic_name__							):


					'''simple docstring'''






					def     __A     (					self     ):
						A__				:       Dict	  =						{}
						A__				:       Tuple	  =						[]
						A__				:       int	  =						1
						A__				:       str	  =						[1, 2]
						A__				:       Union[str, Any]	  =						{"""a""": 1, """b""": 2}
						A__				:       str	  =						{"""a""": [1, 2], """b""": [3, 4]}
						A__				:       int	  =						{"""a""": {"""1""": 1}, """b""": 2}
						A__				:       Any	  =						{"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
						A__				:       List[Any]	  =						{}
						A__				:       List[Any]	  =						[]
						A__				:       Optional[Any]	  =						2
						A__				:       Optional[Any]	  =						[2, 3]
						A__				:       Optional[int]	  =						{"""a""": 2, """b""": 3}
						A__				:       Optional[Any]	  =						{"""a""": [2, 3], """b""": [4, 5]}
						A__				:       List[Any]	  =						{"""a""": {"""1""": 2}, """b""": 3}
						A__				:       Dict	  =						{"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__     )       ,			A__     )

						A__				:       List[str]	  =						2
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(map_nested(A__       ,			A__       ,			num_proc=A__     )       ,			A__     )

						A__				:       Tuple	  =						{"""a""": np.eye(2     ), """b""": np.zeros(3     ), """c""": np.ones(2     )}
						A__				:       Any	  =						{"""a""": 2, """b""": 0, """c""": 2}
						A__				:       List[Any]	  =						{
						    """a""": np.eye(2     ).astype(A__     ),
						    """b""": np.zeros(3     ).astype(A__     ),
						    """c""": np.ones(2     ).astype(A__     ),
						}
						self.assertEqual(map_nested(A__       ,			A__       ,			map_numpy=A__     )       ,			A__     )
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(A__       ,			A__       ,			map_numpy=A__     ).items()}       ,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,			)
						self.assertEqual(map_nested(A__       ,			A__       ,			map_numpy=A__       ,			num_proc=A__     )       ,			A__     )
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(A__       ,			A__       ,			map_numpy=A__       ,			num_proc=A__     ).items()}       ,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,			)
						with self.assertRaises(A__     ):  # can't pickle a local lambda
							map_nested(lambda A__     : x + 1       ,			A__       ,			num_proc=A__     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						{"""a""": 1, """b""": 2}
						A__				:       List[Any]	  =						{"""a""": 3, """b""": 4}
						A__				:       Optional[int]	  =						{"""a""": 5, """b""": 6}
						A__				:       Tuple	  =						sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]     )
						self.assertEqual(sorted(zip_dict(A__       ,			A__       ,			A__     )     )       ,			A__     )






					def     __A     (					self     ):



						class        _a		:


											'''simple docstring'''






											UpperCAmelCase__:  int    				=			'''bar'''



						A__				:       List[str]	  =						Foo()
						self.assertEqual(foo.my_attr       ,			"""bar"""     )
						with temporary_assignment(A__       ,			"""my_attr"""       ,			"""BAR"""     ):
							self.assertEqual(foo.my_attr       ,			"""BAR"""     )
						self.assertEqual(foo.my_attr       ,			"""bar"""     )



@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""       , [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]       , )
def 		UpperCamelCase   (lowercase_:					Union[str, Any]       , lowercase_:					Union[str, Any]       , lowercase_:					List[str]   )	-> Optional[int]:
	with patch("""datasets.utils.py_utils._single_map_nested"""   ) as mock_single_map_nested, patch(
	    """datasets.parallel.parallel.Pool"""   ) as mock_multiprocessing_pool:
		A__				:       Optional[Any]	  =						{f"""{i}""": i for i in range(lowercase_   )}
		A__				:       List[Any]	  =						map_nested(lambda lowercase_   : x + 10       , lowercase_       , num_proc=lowercase_       , parallel_min_length=16   )
		if expected_num_proc == 1:
			assert mock_single_map_nested.called
			assert not mock_multiprocessing_pool.called
		else:
			assert not mock_single_map_nested.called
			assert mock_multiprocessing_pool.called
			assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc



class        _a		(__magic_name__							):


					'''simple docstring'''






					@require_tf
					def     __A     (					self     ):
						import tensorflow as tf
						from tensorflow.keras import layers

						A__				:       str	  =						layers.Dense(2     )

						def gen_random_output():
							A__				:       Dict	  =						tf.random.uniform((1, 3)     )
							return model(A__     ).numpy()

						with temp_seed(42       ,			set_tensorflow=A__     ):
							A__				:       Union[str, Any]	  =						gen_random_output()
						with temp_seed(42       ,			set_tensorflow=A__     ):
							A__				:       str	  =						gen_random_output()
						A__				:       int	  =						gen_random_output()

						np.testing.assert_equal(A__       ,			A__     )
						self.assertGreater(np.abs(outa - outa     ).sum()       ,			0     )






					@require_torch
					def     __A     (					self     ):
						import torch

						def gen_random_output():
							A__				:       str	  =						torch.nn.Linear(3       ,			2     )
							A__				:       Tuple	  =						torch.rand(1       ,			3     )
							return model(A__     ).detach().numpy()

						with temp_seed(42       ,			set_pytorch=A__     ):
							A__				:       Optional[int]	  =						gen_random_output()
						with temp_seed(42       ,			set_pytorch=A__     ):
							A__				:       int	  =						gen_random_output()
						A__				:       List[Any]	  =						gen_random_output()

						np.testing.assert_equal(A__       ,			A__     )
						self.assertGreater(np.abs(outa - outa     ).sum()       ,			0     )






					def     __A     (					self     ):
						def gen_random_output():
							return np.random.rand(1       ,			3     )

						with temp_seed(42     ):
							A__				:       Dict	  =						gen_random_output()
						with temp_seed(42     ):
							A__				:       Tuple	  =						gen_random_output()
						A__				:       Tuple	  =						gen_random_output()

						np.testing.assert_equal(A__       ,			A__     )
						self.assertGreater(np.abs(outa - outa     ).sum()       ,			0     )



@pytest.mark.parametrize("""input_data"""       , [{}]   )
def 		UpperCamelCase   (lowercase_:					List[str]   )	-> Tuple:
	A__				:       Optional[Any]	  =						NestedDataStructure(lowercase_   ).data
	assert output_data == input_data

@pytest.mark.parametrize(
    """data, expected_output"""       , [
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]       , )
def 		UpperCamelCase   (lowercase_:					Optional[int]       , lowercase_:					Optional[Any]   )	-> int:
	A__				:       List[str]	  =						NestedDataStructure(lowercase_   ).flatten()
	assert output == expected_output

def 		UpperCamelCase   ()	-> Any:
	A__				:       Tuple	  =						A(x=1       , y="""foobar"""   )
	A__				:       Optional[int]	  =						{"""x""": 1, """y""": """foobar"""}
	assert asdict(lowercase_   ) == expected_output

	A__				:       Any	  =						{"""a""": {"""b""": A(x=10       , y="""foo"""   )}, """c""": [A(x=20       , y="""bar"""   )]}
	A__				:       Any	  =						{"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
	assert asdict(lowercase_   ) == expected_output

	with pytest.raises(lowercase_   ):
		asdict([1, A(x=10       , y="""foo"""   )]   )

def 		UpperCamelCase   (lowercase_:					str   )	-> Tuple:
	return text.split()

def 		UpperCamelCase   (lowercase_:					Union[str, Any]   )	-> Optional[int]:
	yield (time.time(), content)
	time.sleep(2   )
	yield (time.time(), content)


def 		UpperCamelCase   ()	-> Dict:
	with Pool(2   ) as pool:
		A__				:       str	  =						list(iflatmap_unordered(lowercase_       , _split_text       , kwargs_iterable=[{"""text""": """hello there"""}] * 10   )   )
		assert out.count("""hello"""   ) == 10
		assert out.count("""there"""   ) == 10
		assert len(lowercase_   ) == 20

	# check multiprocess from pathos (uses dill for pickling)
	with multiprocess.Pool(2   ) as pool:
		A__				:       List[Any]	  =						list(iflatmap_unordered(lowercase_       , _split_text       , kwargs_iterable=[{"""text""": """hello there"""}] * 10   )   )
		assert out.count("""hello"""   ) == 10
		assert out.count("""there"""   ) == 10
		assert len(lowercase_   ) == 20

	# check that we get items as fast as possible
	with Pool(2   ) as pool:
		A__				:       int	  =						[]
		for yield_time, content in iflatmap_unordered(
		    lowercase_       , _aseconds_generator_of_aitems_with_timing       , kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]   ):
			assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
			out.append(lowercase_   )
		assert out.count("""a"""   ) == 2
		assert out.count("""b"""   ) == 2
		assert len(lowercase_   ) == 4






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class        _a		(__magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  List[str]    				=			RoCBertTokenizer
					UpperCAmelCase__:  Dict    				=			None
					UpperCAmelCase__:  Optional[Any]    				=			False
					UpperCAmelCase__:  Union[str, Any]    				=			True
					UpperCAmelCase__:  Union[str, Any]    				=			filter_non_english






					def     __A     (					self     ):
						super().setUp()

						A__				:       Tuple	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
						A__				:       Union[str, Any]	  =						{}
						A__				:       Dict	  =						{}
						for i, value in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
							A__				:       Optional[int]	  =						i
						A__				:       Dict	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""vocab_file"""]     )
						A__				:       Optional[Any]	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_shape_file"""]     )
						A__				:       int	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
						with open(self.vocab_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
						with open(self.word_shape_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_shape_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )
						with open(self.word_pronunciation_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_pronunciation_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )






					def     __A     (					self     ):
						A__				:       Dict	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.tokenize("""你好[SEP]你是谁"""     )
						self.assertListEqual(A__       ,			["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )       ,			["""ah""", """\u535A""", """\u63A8""", """zz"""]     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       int	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""h\u00E9llo"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			never_split=["""[UNK]"""]     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )






					def     __A     (					self     ):
						A__				:       List[Any]	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__				:       Any	  =						{}
						for i, token in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
						A__				:       Optional[Any]	  =						RoCBertWordpieceTokenizer(vocab=A__       ,			unk_token="""[UNK]"""     )

						self.assertListEqual(tokenizer.tokenize(""""""     )       ,			[]     )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )       ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )       ,			["""[UNK]""", """runn""", """##ing"""]     )






					def     __A     (					self     ):
						self.assertTrue(_is_whitespace(""" """     )     )
						self.assertTrue(_is_whitespace("""\t"""     )     )
						self.assertTrue(_is_whitespace("""\r"""     )     )
						self.assertTrue(_is_whitespace("""\n"""     )     )
						self.assertTrue(_is_whitespace("""\u00A0"""     )     )

						self.assertFalse(_is_whitespace("""A"""     )     )
						self.assertFalse(_is_whitespace("""-"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_control("""\u0005"""     )     )

						self.assertFalse(_is_control("""A"""     )     )
						self.assertFalse(_is_control(""" """     )     )
						self.assertFalse(_is_control("""\t"""     )     )
						self.assertFalse(_is_control("""\r"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_punctuation("""-"""     )     )
						self.assertTrue(_is_punctuation("""$"""     )     )
						self.assertTrue(_is_punctuation("""`"""     )     )
						self.assertTrue(_is_punctuation("""."""     )     )

						self.assertFalse(_is_punctuation("""A"""     )     )
						self.assertFalse(_is_punctuation(""" """     )     )






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )

						if self.test_rust_tokenizer:
							A__				:       Tuple	  =						self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )






					def     __A     (					self     ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       str	  =						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__				:       Union[str, Any]	  =						tokenizer_r.encode_plus(
								    A__       ,			return_attention_mask=A__       ,			return_token_type_ids=A__       ,			return_offsets_mapping=A__       ,			add_special_tokens=A__       ,			)

								A__				:       Any	  =						tokenizer_r.do_lower_case if hasattr(A__       ,			"""do_lower_case"""     ) else False
								A__				:       List[str]	  =						(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]       ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
								self.assertEqual([e[0] for e in expected_results]       ,			tokens["""offset_mapping"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						["""的""", """人""", """有"""]
						A__				:       List[str]	  =						"""""".join(A__     )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Any	  =						True
								A__				:       int	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[Any]	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Tuple	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       int	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )

								A__				:       Optional[int]	  =						False
								A__				:       Union[str, Any]	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       str	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       Tuple	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Union[str, Any]	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       Tuple	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__				:       str	  =						[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__     )
								]
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )






					@slow
					def     __A     (					self     ):
						A__				:       str	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.encode("""你好"""       ,			add_special_tokens=A__     )
						A__				:       List[Any]	  =						tokenizer.encode("""你是谁"""       ,			add_special_tokens=A__     )

						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__     )
						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__       ,			A__     )

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]






					def     __A     (					self     ):
						A__				:       List[str]	  =						self.get_tokenizers(do_lower_case=A__     )
						for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""     ):
								A__				:       Optional[int]	  =						"""你好，你是谁"""
								A__				:       List[str]	  =						tokenizer.tokenize(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.convert_tokens_to_ids(A__     )
								A__				:       str	  =						tokenizer.convert_tokens_to_shape_ids(A__     )
								A__				:       Optional[int]	  =						tokenizer.convert_tokens_to_pronunciation_ids(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.prepare_for_model(
								    A__       ,			A__       ,			A__       ,			add_special_tokens=A__     )

								A__				:       int	  =						tokenizer.encode_plus(A__       ,			add_special_tokens=A__     )

								self.assertEqual(A__       ,			A__     )



