





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		DebertaTokenizer
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		DebertaTokenizerFast




					def 		lowerCAmelCase     (						self					)						->							Dict:
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										_snake_case													=			[
										    'l',
										    'o',
										    'w',
										    'e',
										    'r',
										    's',
										    't',
										    'i',
										    'd',
										    'n',
										    '\u0120',
										    '\u0120l',
										    '\u0120n',
										    '\u0120lo',
										    '\u0120low',
										    'er',
										    '\u0120lowest',
										    '\u0120newer',
										    '\u0120wider',
										    '[UNK]',
										]
										_snake_case													=			dict(zip(lowerCAmelCase_			,  range(len(lowerCAmelCase_					)					)					)					)
										_snake_case													=			['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
										_snake_case													=			{'unk_token': '[UNK]'}

										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['merges_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write(json.dumps(lowerCAmelCase_					) + '\n'					)
										with open(self.merges_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write('\n'.join(lowerCAmelCase_					)					)




					def 		lowerCAmelCase     (						self			,  **lowerCAmelCase_					)						->							Any:
										kwargs.update(self.special_tokens_map					)
										return self.tokenizer_class.from_pretrained(self.tmpdirname			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Union[str, Any]:
										_snake_case													=			'lower newer'
										_snake_case													=			'lower newer'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			'lower newer'
										_snake_case													=			['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokens + [tokenizer.unk_token]
										_snake_case													=			[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			tokenizer('Hello'			,  'World'					)
										_snake_case													=			[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
										self.assertListEqual(tokd['token_type_ids']			,  lowerCAmelCase_					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			self.tokenizer_class.from_pretrained('microsoft/deberta-base'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(
										    'sequence builders'			,  add_special_tokens=lowerCAmelCase_			,  add_prefix_space=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode(
										    'sequence builders'			,  'multi-sequence build'			,  add_special_tokens=lowerCAmelCase_			,  add_prefix_space=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == encoded_text_from_decode
										assert encoded_pair == encoded_pair_from_decode






					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			[self.tokenizer_class]
										if self.test_rust_tokenizer:
															tokenizer_classes.append(self.rust_tokenizer_class					)

										for tokenizer_class in tokenizer_classes:
															_snake_case													=			tokenizer_class.from_pretrained('microsoft/deberta-base'					)

															_snake_case													=			[
															    'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
															    'ALBERT incorporates two parameter reduction techniques',
															    'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
															    ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
															    ' vocabulary embedding.',
															]

															_snake_case													=			tokenizer(lowerCAmelCase_			,  padding=lowerCAmelCase_					)
															_snake_case													=			[tokenizer.decode(lowerCAmelCase_			,  skip_special_tokens=lowerCAmelCase_					) for seq in encoding['input_ids']]

															# fmt: off
															_snake_case													=			{
															    'input_ids': [
															        [1, 2118, 1_1126, 565, 35, 83, 2_5191, 163, 1_8854, 13, 1_2156, 12, 1_6101, 2_5376, 1_3807, 9, 2_2205, 2_7893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [1, 2118, 1_1126, 565, 2_4536, 80, 4_3797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [1, 133, 78, 65, 16, 10, 3724, 1538, 3_3183, 1_1303, 4_3797, 1938, 4, 870, 2_4165, 2_9105, 5, 739, 3_2644, 3_3183, 1_1303, 3_6173, 88, 80, 650, 7821, 4_5940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 1_3171, 31, 5, 1836, 9, 3_2644, 3_3183, 1_1303, 4, 2]
															    ],
															    'token_type_ids': [
															        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
															    ],
															    'attention_mask': [
															        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
															        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
															    ]
															}
															# fmt: on

															_snake_case													=			[
															    'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
															    'ALBERT incorporates two parameter reduction techniques',
															    'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
															    ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
															    ' vocabulary embedding.',
															]

															self.assertDictEqual(encoding.data			,  lowerCAmelCase_					)

															for expected, decoded in zip(lowerCAmelCase_			,  lowerCAmelCase_					):
																				self.assertEqual(lowerCAmelCase_			,  lowerCAmelCase_					)










import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


UpperCAmelCase_ 							=       [
    {"""dataset""": """wikipedia""", """config_name""": """20220301.de"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.en"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.fr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.frr"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.it"""},
    {"""dataset""": """wikipedia""", """config_name""": """20220301.simple"""},
    {"""dataset""": """snli""", """config_name""": """plain_text"""},
    {"""dataset""": """eli5""", """config_name""": """LFQA_reddit"""},
    {"""dataset""": """wiki40b""", """config_name""": """en"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.compressed"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.nq.no_index"""},
    {"""dataset""": """wiki_dpr""", """config_name""": """psgs_w100.multiset.no_index"""},
    {"""dataset""": """natural_questions""", """config_name""": """default"""},
]





def   lowerCamelCase__       (       UpperCamelCase__     :    Dict=True  )     ->	Dict:



					'''simple docstring'''




					if with_config:
										return [
										    {
										        "testcase_name": d["dataset"] + "/" + d["config_name"],
										        "dataset": d["dataset"],
										        "config_name": d["config_name"],
										    }
										    for d in DATASETS_ON_HF_GCP
										]
					else:
										return [
										    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
										]
@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=_lowerCamelCase   )   )
class      UpperCamelCase_						(					_lowerCamelCase   ):
					lowerCAmelCase_							=		None
					lowerCAmelCase_							=		None




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Union[str, Any]:
										with TemporaryDirectory() as tmp_dir:
															_snake_case													=			dataset_module_factory(lowerCAmelCase_			,  cache_dir=lowerCAmelCase_					)

															_snake_case													=			import_main_class(dataset_module.module_path			,  dataset=lowerCAmelCase_					)

															_snake_case													=			builder_cls(
															    cache_dir=lowerCAmelCase_			,  config_name=lowerCAmelCase_			,  hash=dataset_module.hash			,  )

															_snake_case													=			'/'.join(
															    [
															        HF_GCP_BASE_URL,
															        builder_instance._relative_data_dir(with_hash=lowerCAmelCase_					).replace(os.sep			,  '/'					),
															        config.DATASET_INFO_FILENAME,
															    ]					)
															_snake_case													=			cached_path(lowerCAmelCase_			,  cache_dir=lowerCAmelCase_					)
															self.assertTrue(os.path.exists(lowerCAmelCase_					)					)





@pytest.mark.integration
def   lowerCamelCase__       (       UpperCamelCase__     :    Any  )     ->	Tuple:



					'''simple docstring'''




					_snake_case													=			tmp_path_factory.mktemp('test_hf_gcp'  ) / 'test_wikipedia_simple'
					_snake_case													=			dataset_module_factory('wikipedia'    ,      cache_dir=UpperCamelCase__  )
					_snake_case													=			import_main_class(dataset_module.module_path  )
					_snake_case													=			builder_cls(
					    cache_dir=UpperCamelCase__    ,      config_name='20220301.frr'    ,      hash=dataset_module.hash    ,      )
					# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
					_snake_case													=			None
					builder_instance.download_and_prepare()
					_snake_case													=			builder_instance.as_dataset()
					assert ds





@pytest.mark.integration
def   lowerCamelCase__       (       UpperCamelCase__     :    Union[str, Any]  )     ->	Union[str, Any]:



					'''simple docstring'''




					_snake_case													=			dataset_module_factory('wikipedia'    ,      cache_dir=UpperCamelCase__  )
					_snake_case													=			import_main_class(dataset_module.module_path    ,      dataset=UpperCamelCase__  )
					_snake_case													=			builder_cls(
					    cache_dir=UpperCamelCase__    ,      config_name='20220301.frr'    ,      hash=dataset_module.hash    ,      )
					_snake_case													=			builder_instance.as_streaming_dataset()
					assert ds
					assert isinstance(UpperCamelCase__    ,      UpperCamelCase__  )
					assert "train" in ds
					assert isinstance(ds['train']    ,      UpperCamelCase__  )
					assert next(iter(ds['train']  )  )



