

'''simple docstring'''




import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


UpperCamelCase__ :			str												=				[
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.de'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.en'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.fr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.frr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.it'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.simple'''},
    {'''dataset''': '''snli''', '''config_name''': '''plain_text'''},
    {'''dataset''': '''eli5''', '''config_name''': '''LFQA_reddit'''},
    {'''dataset''': '''wiki40b''', '''config_name''': '''en'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.compressed'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.no_index'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.multiset.no_index'''},
    {'''dataset''': '''natural_questions''', '''config_name''': '''default'''},
]


def        lowerCAmelCase_     (			_lowerCamelCase:     Any=True      ):
		if with_config:
				return [
				    {
				        "testcase_name": d["dataset"] + "/" + d["config_name"],
				        "dataset": d["dataset"],
				        "config_name": d["config_name"],
				    }
				    for d in DATASETS_ON_HF_GCP
				]
		else:
				return [
				    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
				]






@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=lowerCamelCase__					)					)
class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	str    					=	None
			_A   :	Optional[int]    					=	None

			def UpperCamelCase__      (							self  :				Any							,  lowerCAmelCase__  :				List[Any]							,  lowerCAmelCase__  :				Optional[int]		):





					"""simple docstring"""






					with TemporaryDirectory() as tmp_dir:
							__SCREAMING_SNAKE_CASE :	List[Any]					 =	dataset_module_factory(lowerCAmelCase__							,  cache_dir=lowerCAmelCase__		)

							__SCREAMING_SNAKE_CASE :	Optional[int]					 =	import_main_class(dataset_module.module_path							,  dataset=lowerCAmelCase__		)

							__SCREAMING_SNAKE_CASE :	DatasetBuilder					 =	builder_cls(
							    cache_dir=lowerCAmelCase__							,  config_name=lowerCAmelCase__							,  hash=dataset_module.hash							,  )

							__SCREAMING_SNAKE_CASE :	int					 =	"""/""".join(
							    [
							        HF_GCP_BASE_URL,
							        builder_instance._relative_data_dir(with_hash=lowerCAmelCase__		).replace(os.sep							,  """/"""		),
							        config.DATASET_INFO_FILENAME,
							    ]		)
							__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	cached_path(lowerCAmelCase__							,  cache_dir=lowerCAmelCase__		)
							self.assertTrue(os.path.exists(lowerCAmelCase__		)		)


@pytest.mark.integration
def        lowerCAmelCase_     (			_lowerCamelCase:     Any      ):
		__SCREAMING_SNAKE_CASE :	str					 =	tmp_path_factory.mktemp("""test_hf_gcp"""      ) / """test_wikipedia_simple"""
		__SCREAMING_SNAKE_CASE :	List[Any]					 =	dataset_module_factory("""wikipedia""" ,   cache_dir=_lowerCamelCase      )
		__SCREAMING_SNAKE_CASE :	Any					 =	import_main_class(dataset_module.module_path      )
		__SCREAMING_SNAKE_CASE :	DatasetBuilder					 =	builder_cls(
		    cache_dir=_lowerCamelCase ,   config_name="""20220301.frr""" ,   hash=dataset_module.hash ,   )
		# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
		__SCREAMING_SNAKE_CASE :	Any					 =	None
		builder_instance.download_and_prepare()
		__SCREAMING_SNAKE_CASE :	List[Any]					 =	builder_instance.as_dataset()
		assert ds


@pytest.mark.integration
def        lowerCAmelCase_     (			_lowerCamelCase:     List[Any]      ):
		__SCREAMING_SNAKE_CASE :	Any					 =	dataset_module_factory("""wikipedia""" ,   cache_dir=_lowerCamelCase      )
		__SCREAMING_SNAKE_CASE :	List[Any]					 =	import_main_class(dataset_module.module_path ,   dataset=_lowerCamelCase      )
		__SCREAMING_SNAKE_CASE :	DatasetBuilder					 =	builder_cls(
		    cache_dir=_lowerCamelCase ,   config_name="""20220301.frr""" ,   hash=dataset_module.hash ,   )
		__SCREAMING_SNAKE_CASE :	Dict					 =	builder_instance.as_streaming_dataset()
		assert ds
		assert isinstance(_lowerCamelCase ,   _lowerCamelCase      )
		assert "train" in ds
		assert isinstance(ds["""train"""] ,   _lowerCamelCase      )
		assert next(iter(ds["""train"""]      )      )




'''simple docstring'''




# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCamelCase__ :			Tuple												=				'''3'''

print('''Python version:''', sys.version)
print('''transformers version:''', transformers.__version__)

try:
						import torch

						print('''Torch version:''', torch.__version__)
						print('''Cuda available:''', torch.cuda.is_available())
						print('''Cuda version:''', torch.version.cuda)
						print('''CuDNN version:''', torch.backends.cudnn.version())
						print('''Number of GPUs available:''', torch.cuda.device_count())
						print('''NCCL version:''', torch.cuda.nccl.version())
except ImportError:
						print('''Torch version:''', None)

try:
						import deepspeed

						print('''DeepSpeed version:''', deepspeed.__version__)
except ImportError:
						print('''DeepSpeed version:''', None)

try:
						import tensorflow as tf

						print('''TensorFlow version:''', tf.__version__)
						print('''TF GPUs available:''', bool(tf.config.list_physical_devices('''GPU''')))
						print('''Number of TF GPUs available:''', len(tf.config.list_physical_devices('''GPU''')))
except ImportError:
						print('''TensorFlow version:''', None)