


"""simple docstring"""


import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase=None      )		->		Tuple:
				_A         =	None
				# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
				if str(weight.dtype      ) == "torch.bfloat16":
								# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
								_A         =	weight.view(torch.intaa      )
								_A         =	"bfloat16"
				_A         =	weight.cpu().numpy()
				_A         =	os.path.join(__lowercase   ,	f"""{weight_name}.dat"""      )
				if index is not None:
								if dtype is None:
												_A         =	str(array.dtype      )
								_A         =	{"dtype": dtype, "shape": list(array.shape      )}
				if array.ndim == 0:
								_A         =	array[None]
				_A         =	np.memmap(__lowercase   ,	dtype=array.dtype   ,	mode="w+"   ,	shape=array.shape      )
				_A         =	array[:]
				file_array.flush()
				return index






def    a__       ( __lowercase   ,	__lowercase      )		->		Any:
				_A         =	tuple(weight_info["shape"]      )
				if shape == ():
								# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
								_A         =	(1,)

				_A         =	weight_info["dtype"]
				if dtype == "bfloat16":
								# NumPy does not support bfloat16 so this was saved as a int16
								_A         =	"int16"

				_A         =	np.memmap(__lowercase   ,	dtype=__lowercase   ,	shape=__lowercase   ,	mode="r"      )

				if len(weight_info["shape"]      ) == 0:
								_A         =	weight[0]
				_A         =	torch.tensor(__lowercase      )
				if weight_info["dtype"] == "bfloat16":
								_A         =	weight.view(torch.bfloataa      )

				return weight






def    a__       ( __lowercase   ,	__lowercase      )		->		str:
				if index is None or len(__lowercase      ) == 0:
								# Nothing to save
								return

				_A         =	os.path.join(__lowercase   ,	"index.json"      )
				if os.path.isfile(__lowercase      ):
								with open(__lowercase   ,	"r"   ,	encoding="utf-8"      ) as f:
												_A         =	json.load(__lowercase      )
				else:
								_A         =	{}
				current_index.update(__lowercase      )

				with open(__lowercase   ,	"w"   ,	encoding="utf-8"      ) as f:
								json.dump(__lowercase   ,	__lowercase   ,	indent=2      )






def    a__       ( __lowercase   ,	__lowercase      )		->		str:
				os.makedirs(__lowercase   ,	exist_ok=__lowercase      )
				_A         =	{}
				for name, parameter in state_dict.items():
								_A         =	offload_weight(__lowercase   ,	__lowercase   ,	__lowercase   ,	index=__lowercase      )

				# Update index
				save_offload_index(__lowercase   ,	__lowercase      )
class snake_case					(     _UpperCamelCase):







		def __init__(    self					:				List[str]			,      a__					:				Mapping			,      a__					:				str						)				->  str:





						'''simple docstring'''







						_A         =	dataset
						_A         =	prefix







		def __getitem__(    self					:				Tuple			,      a__					:				Optional[int]						)				->  Union[str, Any]:





						'''simple docstring'''







						return self.dataset[F"""{self.prefix}{key}"""]







		def __iter__(    self					:				List[Any]						)				->  List[str]:





						'''simple docstring'''







						return iter([key for key in self.dataset if key.startswith(self.prefix						)]						)







		def __len__(    self					:				Optional[int]						)				->  Optional[int]:





						'''simple docstring'''







						return len(self.dataset						)





class snake_case					(     _UpperCamelCase):







		def __init__(    self					:				Any			,      a__					:				Dict[str, torch.Tensor] = None			,      a__					:				Optional[Union[str, os.PathLike]] = None			,      a__					:				Mapping = None			,      a__					:				Any=None			,      )				->  Optional[int]:





						'''simple docstring'''







						if state_dict is None and save_folder is None:
										raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights."						)

						_A         =	{} if state_dict is None else state_dict
						_A         =	save_folder
						if index is None and save_folder is not None:
										with open(os.path.join(a__			,      "index.json"						)						) as f:
														_A         =	json.load(a__						)
						_A         =	{} if index is None else index
						_A         =	list(self.state_dict.keys()						)
						self.all_keys.extend([key for key in self.index if key not in self.all_keys]						)
						_A         =	device







		def __getitem__(    self					:				Dict			,      a__					:				str						)				->  Dict:





						'''simple docstring'''







						if key in self.state_dict:
										return self.state_dict[key]
						_A         =	self.index[key]
						if weight_info.get("safetensors_file"						) is not None:
										if not is_safetensors_available():
														raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`."						)

										from safetensors import safe_open

										_A         =	"cpu" if self.device is None else self.device
										with safe_open(weight_info["safetensors_file"]			,      framework="pt"			,      device=a__						) as f:
														_A         =	f.get_tensor(weight_info.get("weight_name"			,      a__						)						)

										if "dtype" in weight_info:
														return tensor.to(getattr(a__			,      weight_info["dtype"]						)						)
										else:
														return tensor

						_A         =	os.path.join(self.save_folder			,      F"""{key}.dat"""						)
						return load_offloaded_weight(a__			,      a__						)







		def __iter__(    self					:				int						)				->  Union[str, Any]:





						'''simple docstring'''







						return iter(self.all_keys						)







		def __len__(    self					:				str						)				->  Union[str, Any]:





						'''simple docstring'''







						return len(self.all_keys						)






def    a__       ( __lowercase   ,	__lowercase      )		->		List[str]:
				_A         =	{}
				for module_name in submodule_names:
								# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
								# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
								result.update(
								    {
								        key: param
								        for key, param in state_dict.items()
								        if key == module_name or key.startswith(module_name + "."      )
								    }      )
				return result



"""simple docstring"""


import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


a_											=			get_logger(__name__)


if TYPE_CHECKING:
		from .builder import DatasetBuilder






def    a__       ( __lowercase   ,	__lowercase = None      )		->		Optional[Any]:
				_A         =	importlib.import_module(__lowercase      )

				# TODO(QL): always update the module to add subsequent new authentication
				if hasattr(__lowercase   ,	"_patched_for_streaming"      ) and module._patched_for_streaming:
								return

				def wrap_auth(__lowercase      ):
								@wraps(__lowercase      )
								def wrapper(*__lowercase   ,	**__lowercase      ):
												return function(*__lowercase   ,	use_auth_token=__lowercase   ,	**__lowercase      )

								_A         =	"wrap_auth"
								return wrapper

				# open files in a streaming fashion
				patch_submodule(__lowercase   ,	"open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.listdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.walk"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"glob.glob"   ,	wrap_auth(__lowercase      )      ).start()
				# allow to navigate in remote zip files
				patch_submodule(__lowercase   ,	"os.path.join"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.dirname"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.basename"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.relpath"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.split"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.splitext"   ,	__lowercase      ).start()
				# allow checks on paths
				patch_submodule(__lowercase   ,	"os.path.exists"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isfile"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.getsize"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pathlib.Path"   ,	__lowercase      ).start()
				# file readers
				patch_submodule(__lowercase   ,	"gzip.open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"numpy.load"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_csv"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_excel"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"scipy.io.loadmat"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"xml.etree.ElementTree.parse"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"xml.dom.minidom.parse"   ,	wrap_auth(__lowercase      )      ).start()
				_A         =	True






def    a__       ( __lowercase      )		->		str:
				extend_module_for_streaming(builder.__module__   ,	use_auth_token=builder.use_auth_token      )
				# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
				if not builder.__module__.startswith("datasets."      ):  # check that it's not a packaged builder like csv
								for imports in get_imports(inspect.getfile(builder.__class__      )      ):
												if imports[0] == "internal":
																_A         =	imports[1]
																_A         =	".".join(builder.__module__.split("."      )[:-1] + [internal_import_name]      )
																extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
				from .builder import DatasetBuilder

				_A         =	[
				    cls.__module__
				    for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
				    if issubclass(cls   ,	__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
				]  # check it's not a standard builder from datasets.builder
				for module in parent_builder_modules:
								extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )