
'''simple docstring'''

import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict







@pytest.mark.parametrize(
    "files"    ,						[
        ["full:README.md", "dataset_infos.json"],
        ["empty:README.md", "dataset_infos.json"],
        ["dataset_infos.json"],
        ["full:README.md"],
    ]    ,						)
def 	A  (   UpperCamelCase_       :  Tuple    ,						UpperCamelCase_       :  Optional[Any]   )       ->							List[Any]:







			'''simple docstring'''



			lowerCAmelCase__										=				tmp_path_factory.mktemp("dset_infos_dir"   )
			if "full:README.md" in files:
						with open(dataset_infos_dir / "README.md"    ,						"w"   ) as f:
									f.write("---\ndataset_info:\n  dataset_size: 42\n---"   )
			if "empty:README.md" in files:
						with open(dataset_infos_dir / "README.md"    ,						"w"   ) as f:
									f.write(""   )
    # we want to support dataset_infos.json for backward compatibility
			if "dataset_infos.json" in files:
						with open(dataset_infos_dir / "dataset_infos.json"    ,						"w"   ) as f:
									f.write("{\"default\": {\"dataset_size\": 42}}"   )
			lowerCAmelCase__										=				DatasetInfosDict.from_directory(UpperCamelCase_   )
			assert dataset_infos
			assert dataset_infos["default"].dataset_size == 42







@pytest.mark.parametrize(
    "dataset_info"    ,						[
        DatasetInfo(),
        DatasetInfo(
            description="foo"    ,						features=Features({"a": Value("int32"   )}   )    ,						builder_name="builder"    ,						config_name="config"    ,						version="1.0.0"    ,						splits=[{"name": "train"}]    ,						download_size=42    ,						),
    ]    ,						)
def 	A  (   UpperCamelCase_       :  Optional[Any]    ,						UpperCamelCase_       :  DatasetInfo   )       ->							Optional[int]:







			'''simple docstring'''



			lowerCAmelCase__										=				str(UpperCamelCase_   )
			dataset_info.write_to_directory(UpperCamelCase_   )
			lowerCAmelCase__										=				DatasetInfo.from_directory(UpperCamelCase_   )
			assert dataset_info == reloaded
			assert os.path.exists(os.path.join(UpperCamelCase_    ,						"dataset_info.json"   )   )







def 	A  (   )       ->							List[Any]:







			'''simple docstring'''



			lowerCAmelCase__										=				DatasetInfo(
			    description="foo"    ,						citation="bar"    ,						homepage="https://foo.bar"    ,						license="CC0"    ,						features=Features({"a": Value("int32"   )}   )    ,						post_processed={}    ,						supervised_keys=()    ,						task_templates=[]    ,						builder_name="builder"    ,						config_name="config"    ,						version="1.0.0"    ,						splits=[{"name": "train", "num_examples": 42}]    ,						download_checksums={}    ,						download_size=13_37    ,						post_processing_size=4_42    ,						dataset_size=12_34    ,						size_in_bytes=13_37 + 4_42 + 12_34    ,						)
			lowerCAmelCase__										=				dataset_info._to_yaml_dict()
			assert sorted(UpperCamelCase_   ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML   )
			for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
						assert key in dataset_info_yaml_dict
						assert isinstance(dataset_info_yaml_dict[key]    ,						(list, dict, int, str)   )
			lowerCAmelCase__										=				yaml.safe_dump(UpperCamelCase_   )
			lowerCAmelCase__										=				yaml.safe_load(UpperCamelCase_   )
			assert dataset_info_yaml_dict == reloaded







def 	A  (   )       ->							str:







			'''simple docstring'''



			lowerCAmelCase__										=				DatasetInfo()
			lowerCAmelCase__										=				dataset_info._to_yaml_dict()
			assert dataset_info_yaml_dict == {}







@pytest.mark.parametrize(
    "dataset_infos_dict"    ,						[
        DatasetInfosDict(),
        DatasetInfosDict({"default": DatasetInfo()}   ),
        DatasetInfosDict({"my_config_name": DatasetInfo()}   ),
        DatasetInfosDict(
            {
                "default": DatasetInfo(
                    description="foo"    ,						features=Features({"a": Value("int32"   )}   )    ,						builder_name="builder"    ,						config_name="config"    ,						version="1.0.0"    ,						splits=[{"name": "train"}]    ,						download_size=42    ,						)
            }   ),
        DatasetInfosDict(
            {
                "v1": DatasetInfo(dataset_size=42   ),
                "v2": DatasetInfo(dataset_size=13_37   ),
            }   ),
    ]    ,						)
def 	A  (   UpperCamelCase_       :  Dict    ,						UpperCamelCase_       :  DatasetInfosDict   )       ->							Union[str, Any]:







			'''simple docstring'''



			lowerCAmelCase__										=				str(UpperCamelCase_   )
			dataset_infos_dict.write_to_directory(UpperCamelCase_   )
			lowerCAmelCase__										=				DatasetInfosDict.from_directory(UpperCamelCase_   )

			# the config_name of the dataset_infos_dict take over the attribute
			for config_name, dataset_info in dataset_infos_dict.items():
						lowerCAmelCase__										=				config_name
						# the yaml representation doesn't include fields like description or citation
						# so we just test that we can recover what we can from the yaml
						lowerCAmelCase__										=				DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()   )
			assert dataset_infos_dict == reloaded

			if dataset_infos_dict:
						assert os.path.exists(os.path.join(UpperCamelCase_    ,						"README.md"   )   )









'''simple docstring'''





import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__lowerCamelCase												=						logging.get_logger(__name__)


def        a__     (					):
   # Get the sagemaker specific mp parameters from smp_options variable.
   UpperCAmelCase__	:Any      =      os.getenv('''SM_HP_MP_PARAMETERS''',   '''{}'''						)
   try:
      # Parse it and check the field "partitions" is included, it is required for model parallel.
      UpperCAmelCase__	:Any      =      json.loads(UpperCamelCase_						)
      if "partitions" not in smp_options:
         return False
   except json.JSONDecodeError:
      return False

   # Get the sagemaker specific framework parameters from mpi_options variable.
   UpperCAmelCase__	:Optional[Any]      =      os.getenv('''SM_FRAMEWORK_PARAMS''',   '''{}'''						)
   try:
      # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
      UpperCAmelCase__	:str      =      json.loads(UpperCamelCase_						)
      if not mpi_options.get('''sagemaker_mpi_enabled''',   UpperCamelCase_						):
         return False
   except json.JSONDecodeError:
      return False
   # Lastly, check if the `smdistributed` module is present.
   return importlib.util.find_spec('''smdistributed'''						) is not None


if is_sagemaker_model_parallel_available():
    import smdistributed.modelparallel.torch as smp

    smp.init()




@dataclass
class 				UpperCAmelCase			(			_snake_case						):
 UpperCAmelCase				=					field(
     default=""						, metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}						, )


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[Any] ):
    super().__post_init__()
    warnings.warn(
        '''`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '''
        '''`TrainingArguments` instead.'''			,       __lowerCamelCase			,       )


 @cached_property
 def        __SCREAMING_SNAKE_CASE					(      self							:	List[Any] ):
    logger.info('''PyTorch: setting up devices''' )
    if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
       logger.warning(
           '''torch.distributed process group is initialized, but local_rank == -1. '''
           '''In order to use Torch DDP, launch your script with `python -m torch.distributed.launch''' )
    if self.no_cuda:
       UpperCAmelCase__	:Union[str, Any]      =      torch.device('''cpu''' )
       UpperCAmelCase__	:Dict      =      0
    elif is_sagemaker_model_parallel_available():
       UpperCAmelCase__	:Dict      =      smp.local_rank()
       UpperCAmelCase__	:List[str]      =      torch.device('''cuda'''			,       __lowerCamelCase )
       UpperCAmelCase__	:Optional[int]      =      1
    elif is_sagemaker_dp_enabled():
       import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

       torch.distributed.init_process_group(backend='''smddp'''			,       timeout=self.ddp_timeout_delta )
       UpperCAmelCase__	:int      =      int(os.getenv('''SMDATAPARALLEL_LOCAL_RANK''' ) )
       UpperCAmelCase__	:int      =      torch.device('''cuda'''			,       self.local_rank )
       UpperCAmelCase__	:Union[str, Any]      =      1
    elif self.local_rank == -1:
       # if n_gpu is > 1 we'll use nn.DataParallel.
       # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
       # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
       # trigger an error that a device index is missing. Index 0 takes into account the
       # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
       # will use the first GPU in that env, i.e. GPU#1
       UpperCAmelCase__	:str      =      torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu''' )
       # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
       # the default value.
       UpperCAmelCase__	:str      =      torch.cuda.device_count()
    else:
       # Here, we'll use torch.distributed.
       # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
       if not torch.distributed.is_initialized():
          torch.distributed.init_process_group(backend='''nccl'''			,       timeout=self.ddp_timeout_delta )
       UpperCAmelCase__	:str      =      torch.device('''cuda'''			,       self.local_rank )
       UpperCAmelCase__	:Union[str, Any]      =      1

    if device.type == "cuda":
       torch.cuda.set_device(__lowerCamelCase )

    return device


 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str] ):
    if is_sagemaker_model_parallel_available():
       return smp.dp_size()

    return super().world_size


 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	Dict ):
    return not is_sagemaker_model_parallel_available()





 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	int ):
    return False



