



"""simple docstring"""






import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


__snake_case				:  Any =       [
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.de'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.en'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.fr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.frr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.it'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.simple'''},
    {'''dataset''': '''snli''', '''config_name''': '''plain_text'''},
    {'''dataset''': '''eli5''', '''config_name''': '''LFQA_reddit'''},
    {'''dataset''': '''wiki40b''', '''config_name''': '''en'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.compressed'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.no_index'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.multiset.no_index'''},
    {'''dataset''': '''natural_questions''', '''config_name''': '''default'''},
]


def      _lowercase							(    __snake_case=True      )		->  Dict:
			if with_config:
						return [
						    {
						        "testcase_name": d["dataset"] + "/" + d["config_name"],
						        "dataset": d["dataset"],
						        "config_name": d["config_name"],
						    }
						    for d in DATASETS_ON_HF_GCP
						]
			else:
						return [
						    {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
						]



@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__a				)				)
class      A__  (    __a				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      None
	SCREAMING_SNAKE_CASE        =      None

	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any]			,     _SCREAMING_SNAKE_CASE:						List[Any]			,     _SCREAMING_SNAKE_CASE:						Optional[int])							->			int:

				"""simple docstring"""

				with TemporaryDirectory() as tmp_dir:
							__lowerCAmelCase :    Optional[Any]				=			dataset_module_factory(_lowerCamelCase			,     cache_dir=_lowerCamelCase)

							__lowerCAmelCase :    Optional[Any]				=			import_main_class(dataset_module.module_path			,     dataset=_lowerCamelCase)

							__lowerCAmelCase :    Tuple				=			builder_cls(
							    cache_dir=_lowerCamelCase			,     config_name=_lowerCamelCase			,     hash=dataset_module.hash			,     )

							__lowerCAmelCase :    Union[str, Any]				=			"/".join(
							    [
							        HF_GCP_BASE_URL,
							        builder_instance._relative_data_dir(with_hash=_lowerCamelCase).replace(os.sep			,     "/"),
							        config.DATASET_INFO_FILENAME,
							    ])
							__lowerCAmelCase :    List[Any]				=			cached_path(_lowerCamelCase			,     cache_dir=_lowerCamelCase)
							self.assertTrue(os.path.exists(_lowerCamelCase))






@pytest.mark.integration
def      _lowercase							(    __snake_case      )		->  Dict:
			__lowerCAmelCase :    Optional[int]				=			tmp_path_factory.mktemp("test_hf_gcp"      ) / "test_wikipedia_simple"
			__lowerCAmelCase :    Optional[Any]				=			dataset_module_factory("wikipedia"   ,cache_dir=__snake_case      )
			__lowerCAmelCase :    Any				=			import_main_class(dataset_module.module_path      )
			__lowerCAmelCase :    Dict				=			builder_cls(
			    cache_dir=__snake_case   ,config_name="20220301.frr"   ,hash=dataset_module.hash   ,)
			# use the HF cloud storage, not the original download_and_prepare that uses apache-beam
			__lowerCAmelCase :    List[Any]				=			None
			builder_instance.download_and_prepare()
			__lowerCAmelCase :    List[Any]				=			builder_instance.as_dataset()
			assert ds





@pytest.mark.integration
def      _lowercase							(    __snake_case      )		->  List[str]:
			__lowerCAmelCase :    Union[str, Any]				=			dataset_module_factory("wikipedia"   ,cache_dir=__snake_case      )
			__lowerCAmelCase :    Dict				=			import_main_class(dataset_module.module_path   ,dataset=__snake_case      )
			__lowerCAmelCase :    List[str]				=			builder_cls(
			    cache_dir=__snake_case   ,config_name="20220301.frr"   ,hash=dataset_module.hash   ,)
			__lowerCAmelCase :    Optional[int]				=			builder_instance.as_streaming_dataset()
			assert ds
			assert isinstance(__snake_case   ,__snake_case      )
			assert "train" in ds
			assert isinstance(ds["train"]   ,__snake_case      )
			assert next(iter(ds["train"]      )      )
'''simple docstring'''

import gc
import threading
import time

import psutil
import torch


class     __UpperCAmelCase       :





  def __init__(	self ):
     lowerCAmelCase_			=      psutil.Process()
     lowerCAmelCase_			=      False





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      -1

     while True:
        lowerCAmelCase_			=      max(self.process.memory_info().rss    ,     self.cpu_memory_peak )

        # can't sleep or will not catch the peak right (this comment is here on purpose)
        if not self.peak_monitoring:
           break





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      True
     lowerCAmelCase_			=      threading.Thread(target=self.peak_monitor )
     lowerCAmelCase_			=      True
     self.thread.start()







  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      False
     self.thread.join()
     return self.cpu_memory_peak






A_     :						List[str]    							=PeakCPUMemory()



def snake_case_      (					)			->					Tuple:
   # Time
   lowerCAmelCase_			=      {'''time''': time.time()}

   gc.collect()
   torch.cuda.empty_cache()

   # CPU mem
   lowerCAmelCase_			=      psutil.Process().memory_info().rss
   cpu_peak_tracker.start()

   # GPU mem
   for i in range(torch.cuda.device_count()):
      lowerCAmelCase_			=      torch.cuda.memory_allocated(__snake_case)
   torch.cuda.reset_peak_memory_stats()

   return measures



def snake_case_      (					__snake_case  :							Any)			->					List[str]:
   # Time
   lowerCAmelCase_			=      {'''time''': time.time() - start_measures['''time''']}

   gc.collect()
   torch.cuda.empty_cache()

   # CPU mem
   lowerCAmelCase_			=      (psutil.Process().memory_info().rss - start_measures['''cpu''']) / 2**20
   lowerCAmelCase_			=      (cpu_peak_tracker.stop() - start_measures['''cpu''']) / 2**20

   # GPU mem
   for i in range(torch.cuda.device_count()):
      lowerCAmelCase_			=      (torch.cuda.memory_allocated(__snake_case) - start_measures[str(__snake_case)]) / 2**20
      lowerCAmelCase_			=      (torch.cuda.max_memory_allocated(__snake_case) - start_measures[str(__snake_case)]) / 2**20

   return measures






def snake_case_      (					__snake_case  :							Dict	,   __snake_case  :							Optional[int])			->					Dict:
   print(F'''{description}:''')
   print(F'''- Time: {measures['time']:.2f}s''')
   for i in range(torch.cuda.device_count()):
      print(F'''- GPU {i} allocated: {measures[str(__snake_case)]:.2f}MiB''')
      lowerCAmelCase_			=      measures[F'''{i}-peak''']
      print(F'''- GPU {i} peak: {peak:.2f}MiB''')
   print(F'''- CPU RAM allocated: {measures['cpu']:.2f}MiB''')
   print(F'''- CPU RAM peak: {measures['cpu-peak']:.2f}MiB''')





