
'''simple docstring'''
import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


lowercase__		    =	get_logger(__name__)

# Datasets
lowercase__		    =	"https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
lowercase__		    =	"https://cdn-datasets.huggingface.co/datasets/datasets"
lowercase__		    =	"https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"

# Metrics
lowercase__		    =	"https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"
lowercase__		    =	"https://cdn-datasets.huggingface.co/datasets/metric"
lowercase__		    =	"https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"

# Hub
lowercase__		    =	os.environ.get("HF_ENDPOINT", "https://huggingface.co")
lowercase__		    =	HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
lowercase__		    =	"main"

lowercase__		    =	version.parse(platform.python_version())

if PY_VERSION < version.parse("3.8"):
						import importlib_metadata
else:
						import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
lowercase__		    =	{"1", "ON", "YES", "TRUE"}
lowercase__		    =	ENV_VARS_TRUE_VALUES.union({"AUTO"})


# Imports
lowercase__		    =	version.parse(importlib_metadata.version("dill"))
lowercase__		    =	version.parse(importlib_metadata.version("pandas"))
lowercase__		    =	version.parse(importlib_metadata.version("pyarrow"))

lowercase__		    =	os.environ.get("USE_TF", "AUTO").upper()
lowercase__		    =	os.environ.get("USE_TORCH", "AUTO").upper()
lowercase__		    =	os.environ.get("USE_JAX", "AUTO").upper()

lowercase__		    =	"N/A"
lowercase__		    =	False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
						lowercase__		    =	importlib.util.find_spec("torch") is not None
						if TORCH_AVAILABLE:
												try:
																		lowercase__		    =	version.parse(importlib_metadata.version("torch"))
																		logger.info(f"PyTorch version {TORCH_VERSION} available.")
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling PyTorch because USE_TF is set")

lowercase__		    =	"N/A"
lowercase__		    =	False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
						lowercase__		    =	importlib.util.find_spec("tensorflow") is not None
						if TF_AVAILABLE:
												# For the metadata, we have to look for both tensorflow and tensorflow-cpu
												for package in [
												    "tensorflow",
												    "tensorflow-cpu",
												    "tensorflow-gpu",
												    "tf-nightly",
												    "tf-nightly-cpu",
												    "tf-nightly-gpu",
												    "intel-tensorflow",
												    "tensorflow-rocm",
												    "tensorflow-macos",
												]:
																		try:
																								lowercase__		    =	version.parse(importlib_metadata.version(package))
																		except importlib_metadata.PackageNotFoundError:
																								continue
																		else:
																								break
												else:
																		lowercase__		    =	False
						if TF_AVAILABLE:
												if TF_VERSION.major < 2:
																		logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
																		lowercase__		    =	False
												else:
																		logger.info(f"TensorFlow version {TF_VERSION} available.")
else:
						logger.info("Disabling Tensorflow because USE_TORCH is set")


lowercase__		    =	"N/A"
lowercase__		    =	False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
						lowercase__		    =	importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
						if JAX_AVAILABLE:
												try:
																		lowercase__		    =	version.parse(importlib_metadata.version("jax"))
																		logger.info(f"JAX version {JAX_VERSION} available.")
												except importlib_metadata.PackageNotFoundError:
																		pass
else:
						logger.info("Disabling JAX because USE_JAX is set to False")


lowercase__		    =	os.environ.get("USE_BEAM", "AUTO").upper()
lowercase__		    =	"N/A"
lowercase__		    =	False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
						try:
												lowercase__		    =	version.parse(importlib_metadata.version("apache_beam"))
												lowercase__		    =	True
												logger.info(f"Apache Beam version {BEAM_VERSION} available.")
						except importlib_metadata.PackageNotFoundError:
												pass
else:
						logger.info("Disabling Apache Beam because USE_BEAM is set to False")


# Optional tools for data loading
lowercase__		    =	importlib.util.find_spec("sqlalchemy") is not None

# Optional tools for feature decoding
lowercase__		    =	importlib.util.find_spec("PIL") is not None
lowercase__		    =	importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.0.31")
lowercase__		    =	importlib.util.find_spec("soundfile") is not None and version.parse(
    importlib.import_module("soundfile").__libsndfile_version__
) >= version.parse("1.1.0")

# Optional compression tools
lowercase__		    =	importlib.util.find_spec("rarfile") is not None
lowercase__		    =	importlib.util.find_spec("zstandard") is not None
lowercase__		    =	importlib.util.find_spec("lz4") is not None
lowercase__		    =	importlib.util.find_spec("py7zr") is not None


# Cache location
lowercase__		    =	"~/.cache"
lowercase__		    =	os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
lowercase__		    =	os.path.join(XDG_CACHE_HOME, "huggingface")
lowercase__		    =	os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))

lowercase__		    =	os.path.join(HF_CACHE_HOME, "datasets")
lowercase__		    =	Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))

lowercase__		    =	os.path.join(HF_CACHE_HOME, "metrics")
lowercase__		    =	Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE))

lowercase__		    =	os.path.join(HF_CACHE_HOME, "modules")
lowercase__		    =	Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))

lowercase__		    =	"downloads"
lowercase__		    =	os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
lowercase__		    =	Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))

lowercase__		    =	"extracted"
lowercase__		    =	os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
lowercase__		    =	Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
lowercase__		    =	(
    os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
lowercase__		    =	1_0_0_0

# Size of the preloaded record batch in `Dataset.__iter__`
lowercase__		    =	1_0

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
lowercase__		    =	4 << 3_0

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
lowercase__		    =	"500MB"

# Parquet configuration
lowercase__		    =	1_0_0
lowercase__		    =	1_0_0
lowercase__		    =	1_0_0

# Offline mode
lowercase__		    =	os.environ.get("HF_DATASETS_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES

# In-memory
lowercase__		    =	0  # Disabled
lowercase__		    =	float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
lowercase__		    =	"dataset.arrow"
lowercase__		    =	"indices.arrow"
lowercase__		    =	"state.json"
lowercase__		    =	"dataset_info.json"
lowercase__		    =	"dataset_infos.json"
lowercase__		    =	"LICENSE"
lowercase__		    =	"metric_info.json"
lowercase__		    =	"dataset_dict.json"

lowercase__		    =	"datasets_modules"

lowercase__		    =	2_5_5

# Streaming
lowercase__		    =	2_0
lowercase__		    =	5

# Datasets without script
lowercase__		    =	2_0_0
lowercase__		    =	1_0
lowercase__		    =	2_0_0

# Progress bars
lowercase__		    =	0.05  # 20 progress updates per sec




'''simple docstring'''
#
# Sort large text files in a minimum amount of memory
#
import argparse
import os






class      snake_case__    :







		"""simple docstring"""

		lowerCamelCase							=		"""block_{0}.dat"""



		def __init__(	self							:       List[str]   ,     UpperCamelCase__							:       str	)		->       Union[str, Any]:






				"""simple docstring"""







				snake_case   :    Optional[int]			       =		filename
				snake_case   :    Any			       =		[]



		def 	lowerCAmelCase							(	self							:       Any   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       str	)		->       Dict:






				"""simple docstring"""







				snake_case   :    str			       =		self.BLOCK_FILENAME_FORMAT.format(UpperCamelCase__	)
				with open(UpperCamelCase__   ,     '''w'''	) as file:
						file.write(UpperCamelCase__	)
				self.block_filenames.append(UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       Optional[Any]	)		->       Optional[int]:






				"""simple docstring"""







				return self.block_filenames



		def 	lowerCAmelCase							(	self							:       Optional[int]   ,     UpperCamelCase__							:       Union[str, Any]   ,     UpperCamelCase__							:       Optional[Any]=None	)		->       Dict:






				"""simple docstring"""







				snake_case   :    Optional[Any]			       =		0
				with open(self.filename	) as file:
						while True:
								snake_case   :    List[str]			       =		file.readlines(UpperCamelCase__	)

								if lines == []:
										break

								if sort_key is None:
										lines.sort()
								else:
										lines.sort(key=UpperCamelCase__	)

								self.write_block(''''''.join(UpperCamelCase__	)   ,     UpperCamelCase__	)
								i += 1



		def 	lowerCAmelCase							(	self							:       Dict	)		->       Optional[Any]:






				"""simple docstring"""







				map(os.remove   ,     self.block_filenames	)







class      snake_case__    :







		"""simple docstring"""

		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       Optional[int]	)		->       Dict:






				"""simple docstring"""







				snake_case   :    str			       =		-1
				snake_case   :    Optional[Any]			       =		None

				for i in range(len(UpperCamelCase__	)	):
						if min_str is None or choices[i] < min_str:
								snake_case   :    Union[str, Any]			       =		i

				return min_index







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       str   ,     UpperCamelCase__							:       Tuple	)		->       Dict:






				"""simple docstring"""







				snake_case   :    Dict			       =		files
				snake_case   :    Optional[Any]			       =		set()
				snake_case   :    Tuple			       =		len(UpperCamelCase__	)
				snake_case   :    List[Any]			       =		{i: None for i in range(self.num_buffers	)}



		def 	lowerCAmelCase							(	self							:       Tuple	)		->       int:






				"""simple docstring"""







				return {
				    i: self.buffers[i] for i in range(self.num_buffers	) if i not in self.empty
				}



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       int:






				"""simple docstring"""







				for i in range(self.num_buffers	):
						if self.buffers[i] is None and i not in self.empty:
								snake_case   :    Optional[Any]			       =		self.files[i].readline()

								if self.buffers[i] == "":
										self.empty.add(UpperCamelCase__	)
										self.files[i].close()

				if len(self.empty	) == self.num_buffers:
						return False

				return True



		def 	lowerCAmelCase							(	self							:       Optional[Any]   ,     UpperCamelCase__							:       Any	)		->       str:






				"""simple docstring"""







				snake_case   :    List[Any]			       =		self.buffers[index]
				snake_case   :    Optional[int]			       =		None

				return value







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       List[Any]   ,     UpperCamelCase__							:       Optional[Any]	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    Optional[int]			       =		merge_strategy



		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       Optional[int]   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       Dict	)		->       Tuple:






				"""simple docstring"""







				snake_case   :    List[Any]			       =		FilesArray(self.get_file_handles(UpperCamelCase__   ,     UpperCamelCase__	)	)
				with open(UpperCamelCase__   ,     '''w'''   ,     UpperCamelCase__	) as outfile:
						while buffers.refresh():
								snake_case   :    List[Any]			       =		self.merge_strategy.select(buffers.get_dict()	)
								outfile.write(buffers.unshift(UpperCamelCase__	)	)



		def 	lowerCAmelCase							(	self							:       List[str]   ,     UpperCamelCase__							:       Tuple   ,     UpperCamelCase__							:       str	)		->       int:






				"""simple docstring"""







				snake_case   :    str			       =		{}

				for i in range(len(UpperCamelCase__	)	):
						snake_case   :    Any			       =		open(filenames[i]   ,     '''r'''   ,     UpperCamelCase__	)  # noqa: UP015

				return files







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       Any   ,     UpperCamelCase__							:       Any	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    Optional[Any]			       =		block_size



		def 	lowerCAmelCase							(	self							:       Dict   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       Tuple=None	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    List[str]			       =		self.get_number_blocks(UpperCamelCase__   ,     self.block_size	)
				snake_case   :    Optional[Any]			       =		FileSplitter(UpperCamelCase__	)
				splitter.split(self.block_size   ,     UpperCamelCase__	)

				snake_case   :    Optional[Any]			       =		FileMerger(NWayMerge()	)
				snake_case   :    Any			       =		self.block_size / (num_blocks + 1)
				merger.merge(splitter.get_block_filenames()   ,     filename + '''.out'''   ,     UpperCamelCase__	)

				splitter.cleanup()



		def 	lowerCAmelCase							(	self							:       Optional[Any]   ,     UpperCamelCase__							:       List[str]   ,     UpperCamelCase__							:       Dict	)		->       List[Any]:






				"""simple docstring"""







				return (os.stat(UpperCamelCase__	).st_size / block_size) + 1







def 					_UpperCamelCase		(    SCREAMING_SNAKE_CASE__       )   ->				List[Any]:


		'''simple docstring'''


		if string[-1].lower() == "k":
				return int(string[:-1]       ) * 1024
		elif string[-1].lower() == "m":
				return int(string[:-1]       ) * 1024 * 1024
		elif string[-1].lower() == "g":
				return int(string[:-1]       ) * 1024 * 1024 * 1024
		else:
				return int(SCREAMING_SNAKE_CASE__       )




def 					_UpperCamelCase		(    )   ->				Optional[int]:


		'''simple docstring'''


		snake_case   :    Tuple			       =		argparse.ArgumentParser()
		parser.add_argument(
		    '''-m'''      ,    '''--mem'''      ,    help='''amount of memory to use for sorting'''      ,    default='''100M'''       )
		parser.add_argument(
		    '''filename'''      ,    metavar='''<filename>'''      ,    nargs=1      ,    help='''name of file to sort'''       )
		snake_case   :    Any			       =		parser.parse_args()

		snake_case   :    int			       =		ExternalSort(parse_memory(args.mem       )       )
		sorter.sort(args.filename[0]       )


if __name__ == "__main__":
						main()


