
import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


UpperCamelCase_   	=	"<<<<<<< This should probably be modified because it mentions: "

UpperCamelCase_   	=	"=======\n>>>>>>>\n"

UpperCamelCase_   	=	[
    "TextEncoderConfig",
    "ByteTextEncoder",
    "SubwordTextEncoder",
    "encoder_config",
    "maybe_build_from_corpus",
    "manual_dir",
]

UpperCamelCase_   	=	[
    # (pattern, replacement)
    # Order is important here for some replacements
    (R"tfds\.core", R"datasets"),
    (R"tf\.io\.gfile\.GFile", R"open"),
    (R"tf\.([\w\d]+)", R"datasets.Value('\1')"),
    (R"tfds\.features\.Text\(\)", R"datasets.Value('string')"),
    (R"tfds\.features\.Text\(", R"datasets.Value('string'),"),
    (R"features\s*=\s*tfds.features.FeaturesDict\(", R"features=datasets.Features("),
    (R"tfds\.features\.FeaturesDict\(", R"dict("),
    (R"The TensorFlow Datasets Authors", R"The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"),
    (R"tfds\.", R"datasets."),
    (R"dl_manager\.manual_dir", R"self.config.data_dir"),
    (R"self\.builder_config", R"self.config"),
]






def 		_UpperCAmelCase   (	UpperCamelCase:     Namespace  ):






	"""simple docstring"""


	return ConvertCommand(args.tfds_path			,							args.datasets_directory  )







class       a		(   __UpperCAmelCase					):







							@staticmethod
							def       UpperCAmelCase__					(						snake_case__  :  ArgumentParser ):
								"""simple docstring"""







								__lowerCAmelCase         =							parser.add_parser(
								    "convert"      ,				help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset."      ,				)
								train_parser.add_argument(
								    "--tfds_path"      ,				type=snake_case__      ,				required=snake_case__      ,				help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert."      ,				)
								train_parser.add_argument(
								    "--datasets_directory"      ,				type=snake_case__      ,				required=snake_case__      ,				help="Path to the HuggingFace Datasets folder." )
								train_parser.set_defaults(func=snake_case__ )







							def __init__(						self  :  List[Any]      ,				snake_case__  :  str      ,				snake_case__  :  str      ,				*snake_case__  :  str ):
								"""simple docstring"""







								__lowerCAmelCase         =							get_logger("datasets-cli/converting" )

								__lowerCAmelCase         =							tfds_path
								__lowerCAmelCase         =							datasets_directory







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								if os.path.isdir(self._tfds_path ):
									__lowerCAmelCase         =							os.path.abspath(self._tfds_path )
								elif os.path.isfile(self._tfds_path ):
									__lowerCAmelCase         =							os.path.dirname(self._tfds_path )
								else:
									raise ValueError("--tfds_path is neither a directory nor a file. Please check path." )

								__lowerCAmelCase         =							os.path.abspath(self._datasets_directory )

								self._logger.info(F"Converting datasets from {abs_tfds_path} to {abs_datasets_path}" )

								__lowerCAmelCase         =							[]
								__lowerCAmelCase         =							[]
								__lowerCAmelCase         =							{}

								if os.path.isdir(self._tfds_path ):
									__lowerCAmelCase         =							os.listdir(snake_case__ )
								else:
									__lowerCAmelCase         =							[os.path.basename(self._tfds_path )]

								for f_name in file_names:
									self._logger.info(F"Looking at file {f_name}" )
									__lowerCAmelCase         =							os.path.join(snake_case__      ,				snake_case__ )
									__lowerCAmelCase         =							os.path.join(snake_case__      ,				snake_case__ )

									if not os.path.isfile(snake_case__ ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
										self._logger.info("Skipping file" )
										continue

									with open(snake_case__      ,				encoding="utf-8" ) as f:
										__lowerCAmelCase         =							f.readlines()

									__lowerCAmelCase         =							[]
									__lowerCAmelCase         =							False
									__lowerCAmelCase         =							False
									__lowerCAmelCase         =							[]
									for line in lines:
										__lowerCAmelCase         =							line

										# Convert imports
										if "import tensorflow.compat.v2 as tf" in out_line:
											continue
										elif "@tfds.core" in out_line:
											continue
										elif "builder=self" in out_line:
											continue
										elif "import tensorflow_datasets.public_api as tfds" in out_line:
											__lowerCAmelCase         =							"import datasets\n"
										elif "import tensorflow" in out_line:
											# order is important here
											__lowerCAmelCase         =							""
											continue
										elif "from absl import logging" in out_line:
											__lowerCAmelCase         =							"from datasets import logging\n"
										elif "getLogger" in out_line:
											__lowerCAmelCase         =							out_line.replace("getLogger"      ,				"get_logger" )
										elif any(expression in out_line for expression in TO_HIGHLIGHT ):
											__lowerCAmelCase         =							True
											__lowerCAmelCase         =							list(filter(lambda snake_case__ : e in out_line      ,				snake_case__ ) )
											out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(snake_case__ ) + "\n" )
											out_lines.append(snake_case__ )
											out_lines.append(snake_case__ )
											continue
										else:
											for pattern, replacement in TO_CONVERT:
												__lowerCAmelCase         =							re.sub(snake_case__      ,				snake_case__      ,				snake_case__ )

                # Take care of saving utilities (to later move them together with main script)
										if "tensorflow_datasets" in out_line:
											__lowerCAmelCase         =							re.match(R"from\stensorflow_datasets.*import\s([^\.\r\n]+)"      ,				snake_case__ )
											tfds_imports.extend(imp.strip() for imp in match.group(1 ).split("," ) )
											__lowerCAmelCase         =							"from . import " + match.group(1 )

										# Check we have not forget anything
										if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
											raise ValueError(F"Error converting {out_line.strip()}" )

										if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
											__lowerCAmelCase         =							True
										out_lines.append(snake_case__ )

									if is_builder or "wmt" in f_name:
										# We create a new directory for each dataset
										__lowerCAmelCase         =							f_name.replace(".py"      ,				"" )
										__lowerCAmelCase         =							os.path.join(snake_case__      ,				snake_case__ )
										__lowerCAmelCase         =							os.path.join(snake_case__      ,				snake_case__ )
										os.makedirs(snake_case__      ,				exist_ok=snake_case__ )
										self._logger.info(F"Adding directory {output_dir}" )
										imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
									else:
										# Utilities will be moved at the end
										utils_files.append(snake_case__ )

									if needs_manual_update:
										with_manual_update.append(snake_case__ )

									with open(snake_case__      ,				"w"      ,				encoding="utf-8" ) as f:
										f.writelines(snake_case__ )
									self._logger.info(F"Converted in {output_file}" )

								for utils_file in utils_files:
									try:
										__lowerCAmelCase         =							os.path.basename(snake_case__ )
										__lowerCAmelCase         =							imports_to_builder_map[f_name.replace(".py"      ,				"" )]
										self._logger.info(F"Moving {dest_folder} to {utils_file}" )
										shutil.copy(snake_case__      ,				snake_case__ )
									except KeyError:
										self._logger.error(F"Cannot find destination folder for {utils_file}. Please copy manually." )

								if with_manual_update:
									for file_path in with_manual_update:
										self._logger.warning(
										    F"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." )








import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_   	=	get_tests_dir("fixtures/spiece.model")

@require_sentencepiece
@require_tokenizers
class       a		(   __UpperCAmelCase      ,	unittest.TestCase					):
							lowercase_		:						Union[str, Any]   							=  DebertaVaTokenizer
							lowercase_		:						List[str]   							=  DebertaVaTokenizerFast
							lowercase_		:						str   							=  True
							lowercase_		:						List[Any]   							=  True







							def       UpperCAmelCase__					(						self  :  Optional[Any] ):
								"""simple docstring"""







								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				unk_token="<unk>" )
								tokenizer.save_pretrained(self.tmpdirname )







							def       UpperCAmelCase__					(						self  :  str      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							"this is a test"
								__lowerCAmelCase         =							"this is a test"
								return input_text, output_text







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							"<pad>"
								__lowerCAmelCase         =							0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__ )      ,				snake_case__ )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__ )      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							list(self.get_tokenizer().get_vocab().keys() )
								self.assertEqual(vocab_keys[0]      ,				"<pad>" )
								self.assertEqual(vocab_keys[1]      ,				"<unk>" )
								self.assertEqual(vocab_keys[-1]      ,				"[PAD]" )
								self.assertEqual(len(snake_case__ )      ,				30_001 )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								self.assertEqual(self.get_tokenizer().vocab_size      ,				30_000 )







							def       UpperCAmelCase__					(						self  :  List[str] ):
								"""simple docstring"""







								__lowerCAmelCase         =							" \tHeLLo!how  \n Are yoU?  "
								__lowerCAmelCase         =							["▁hello", "!", "how", "▁are", "▁you", "?"]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one." )
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								pass







							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one." )
							def       UpperCAmelCase__					(						self  :  List[str] ):
								"""simple docstring"""







								pass







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							" \tHeLLo!how  \n Are yoU?  "
								__lowerCAmelCase         =							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.get_tokenizer()
								__lowerCAmelCase         =							self.get_rust_tokenizer()

								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."

								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							self.get_rust_tokenizer()
								__lowerCAmelCase         =							tokenizer.encode(snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							"This is a test"
								__lowerCAmelCase         =							[13, 1, 4_398, 25, 21, 1_289]
								__lowerCAmelCase         =							["▁", "T", "his", "▁is", "▁a", "▁test"]
								__lowerCAmelCase         =							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				keep_accents=snake_case__ )
								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				keep_accents=snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								# fmt: off
								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
								__lowerCAmelCase         =							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode("sequence builders" )
								__lowerCAmelCase         =							tokenizer.encode("multi-sequence build" )

								__lowerCAmelCase         =							tokenizer.build_inputs_with_special_tokens(snake_case__ )
								__lowerCAmelCase         =							tokenizer.build_inputs_with_special_tokens(snake_case__      ,				snake_case__ )

								self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]      ,				snake_case__ )
								self.assertEqual(
								    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]      ,				snake_case__      ,				)







							@slow
							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case__      ,				model_name="microsoft/deberta-v2-xlarge"      ,				revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"      ,				)






