



'''simple docstring'''
# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


snake_case_					:      Any    			=    get_logger(__name__)






class 				lowercase__      :
	lowercase__    							=     """dummy_data"""
	lowercase__    							=     """datasets"""
	lowercase__    							=     False





	def __init__(					self   :							Tuple					,lowerCamelCase__   :							str					,lowerCamelCase__   :							str					,lowerCamelCase__   :							Union[Version, str]					,lowerCamelCase__   :							Optional[str] = None					,lowerCamelCase__   :							bool = False					,lowerCamelCase__   :							bool = True					,lowerCamelCase__   :							Optional[List[Callable]] = None					,):







							'''simple docstring'''
							_UpperCamelCase   :						Tuple												=  0
							_UpperCamelCase   :						Union[str, Any]												=  dataset_name
							_UpperCamelCase   :						str												=  cache_dir
							_UpperCamelCase   :						int												=  use_local_dummy_data
							_UpperCamelCase   :						Any												=  config
							# download_callbacks take a single url as input
							_UpperCamelCase   :						List[Callable]												=  download_callbacks or []
							# if False, it doesn't load existing files and it returns the paths of the dummy files relative
							# to the dummy_data zip file root
							_UpperCamelCase   :						Optional[Any]												=  load_existing_dummy_data

							# TODO(PVP, QL) might need to make this more general
							_UpperCamelCase   :						Tuple												=  str(lowerCamelCase__					)
							# to be downloaded
							_UpperCamelCase   :						str												=  None
							_UpperCamelCase   :						Any												=  None





	@property
	def       UpperCamelCase_    (					self   :							Optional[Any]					):







							'''simple docstring'''
							if self._dummy_file is None:
													_UpperCamelCase   :						Any												=  self.download_dummy_data()
							return self._dummy_file





	@property
	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							if self.config is not None:
													# structure is dummy / config_name / version_name
													return os.path.join('dummy'					,self.config.name					,self.version_name					)
							# structure is dummy / version_name
							return os.path.join('dummy'					,self.version_name					)





	@property
	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							return os.path.join(self.dummy_data_folder					,'dummy_data.zip'					)





	def       UpperCamelCase_    (					self   :							Any					):







							'''simple docstring'''
							_UpperCamelCase   :						List[Any]												=  (
							    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
							)

							_UpperCamelCase   :						int												=  cached_path(
							    lowerCamelCase__					,cache_dir=self.cache_dir					,extract_compressed_file=lowerCamelCase__					,force_extract=lowerCamelCase__					)

							return os.path.join(lowerCamelCase__					,self.dummy_file_name					)





	@property
	def       UpperCamelCase_    (					self   :							Any					):







							'''simple docstring'''
							return os.path.join(self.datasets_scripts_dir					,self.dataset_name					,self.dummy_zip_file					)





	@property
	def       UpperCamelCase_    (					self   :							Optional[Any]					):







							'''simple docstring'''
							if self._bucket_url is None:
													_UpperCamelCase   :						List[str]												=  hf_github_url(self.dataset_name					,self.dummy_zip_file.replace(os.sep					,'/'					)					)
							return self._bucket_url





	@property
	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							# return full path if its a dir
							if os.path.isdir(self.dummy_file					):
													return self.dummy_file
							# else cut off path to file -> example `xsum`.
							return "/".join(self.dummy_file.replace(os.sep					,'/'					).split('/'					)[:-1]					)





	def       UpperCamelCase_    (					self   :							Dict					,lowerCamelCase__   :							Any					,*lowerCamelCase__   :							str					):







							'''simple docstring'''
							if self.load_existing_dummy_data:
													# dummy data is downloaded and tested
													_UpperCamelCase   :						Optional[Any]												=  self.dummy_file
							else:
													# dummy data cannot be downloaded and only the path to dummy file is returned
													_UpperCamelCase   :						Tuple												=  self.dummy_file_name

							# special case when data_url is a dict
							if isinstance(lowerCamelCase__					,lowerCamelCase__					):
													return self.create_dummy_data_dict(lowerCamelCase__					,lowerCamelCase__					)
							elif isinstance(lowerCamelCase__					,(list, tuple)					):
													return self.create_dummy_data_list(lowerCamelCase__					,lowerCamelCase__					)
							else:
													return self.create_dummy_data_single(lowerCamelCase__					,lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							Optional[int]					,lowerCamelCase__   :							int					,*lowerCamelCase__   :							Tuple					):







							'''simple docstring'''
							return self.download_and_extract(lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							Optional[int]					,lowerCamelCase__   :							Optional[int]					,lowerCamelCase__   :							int					):







							'''simple docstring'''
							return self.download_and_extract(lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							Optional[int]					,lowerCamelCase__   :							int					,*lowerCamelCase__   :							Tuple					,**lowerCamelCase__   :							str					):







							'''simple docstring'''
							return path





	def       UpperCamelCase_    (					self   :							Optional[int]					):







							'''simple docstring'''
							return {}





	def       UpperCamelCase_    (					self   :							Tuple					,lowerCamelCase__   :							List[Any]					,lowerCamelCase__   :							Dict					):







							'''simple docstring'''
							_UpperCamelCase   :						List[Any]												=  {}
							for key, single_urls in data_url.items():
													for download_callback in self.download_callbacks:
																			if isinstance(lowerCamelCase__					,lowerCamelCase__					):
																									for single_url in single_urls:
																															download_callback(lowerCamelCase__					)
																			else:
																									_UpperCamelCase   :						Tuple												=  single_urls
																									download_callback(lowerCamelCase__					)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
													if isinstance(lowerCamelCase__					,lowerCamelCase__					):
																			_UpperCamelCase   :						Dict												=  [os.path.join(lowerCamelCase__					,urllib.parse.quote_plus(Path(lowerCamelCase__					).name					)					) for x in single_urls]
													else:
																			_UpperCamelCase   :						str												=  single_urls
																			_UpperCamelCase   :						List[Any]												=  os.path.join(lowerCamelCase__					,urllib.parse.quote_plus(Path(lowerCamelCase__					).name					)					)
													_UpperCamelCase   :						int												=  value

							# make sure that values are unique
							if all(isinstance(lowerCamelCase__					,lowerCamelCase__					) for i in dummy_data_dict.values()					) and len(set(dummy_data_dict.values()					)					) < len(
							    dummy_data_dict.values()					):
													# append key to value to make its name unique
													_UpperCamelCase   :						Optional[Any]												=  {key: value + key for key, value in dummy_data_dict.items()}

							return dummy_data_dict





	def       UpperCamelCase_    (					self   :							Union[str, Any]					,lowerCamelCase__   :							List[str]					,lowerCamelCase__   :							str					):







							'''simple docstring'''
							_UpperCamelCase   :						Dict												=  []
							# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
							_UpperCamelCase   :						str												=  all(bool(re.findall('[0-9]{3,}-of-[0-9]{3,}'					,lowerCamelCase__					)					) for url in data_url					)
							_UpperCamelCase   :						Union[str, Any]												=  all(
							    url.startswith('https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed'					) for url in data_url					)
							if data_url and (is_tf_records or is_pubmed_records):
													_UpperCamelCase   :						Optional[Any]												=  [data_url[0]] * len(lowerCamelCase__					)
							for single_url in data_url:
													for download_callback in self.download_callbacks:
																			download_callback(lowerCamelCase__					)
													# we force the name of each key to be the last file / folder name of the url path
													# if the url has arguments, we need to encode them with urllib.parse.quote_plus
													_UpperCamelCase   :						Dict												=  os.path.join(lowerCamelCase__					,urllib.parse.quote_plus(single_url.split('/'					)[-1]					)					)
													dummy_data_list.append(lowerCamelCase__					)
							return dummy_data_list





	def       UpperCamelCase_    (					self   :							Tuple					,lowerCamelCase__   :							List[Any]					,lowerCamelCase__   :							Any					):







							'''simple docstring'''
							for download_callback in self.download_callbacks:
													download_callback(lowerCamelCase__					)
							# we force the name of each key to be the last file / folder name of the url path
							# if the url has arguments, we need to encode them with urllib.parse.quote_plus
							_UpperCamelCase   :						List[Any]												=  os.path.join(lowerCamelCase__					,urllib.parse.quote_plus(data_url.split('/'					)[-1]					)					)
							if os.path.exists(lowerCamelCase__					) or not self.load_existing_dummy_data:
													return value
							else:
													# Backward compatibility, maybe deprecate at one point.
													# For many datasets with single url calls to dl_manager.download_and_extract,
													# the dummy_data.zip file is actually the zipped downloaded file
													# while now we expected the dummy_data.zip file to be a directory containing
													# the downloaded file.
													return path_to_dummy_data





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							pass





	def       UpperCamelCase_    (					self   :							int					):







							'''simple docstring'''
							pass





	def       UpperCamelCase_    (					self   :							Optional[int]					,lowerCamelCase__   :							Optional[Any]					):







							'''simple docstring'''
							def _iter_archive_members(lowerCamelCase__   :							Optional[Any]					):
													# this preserves the order of the members inside the ZIP archive
													_UpperCamelCase   :						List[Any]												=  Path(self.dummy_file					).parent
													_UpperCamelCase   :						Any												=  path.relative_to(lowerCamelCase__					)
													with ZipFile(self.local_path_to_dummy_data					) as zip_file:
																			_UpperCamelCase   :						int												=  zip_file.namelist()
													for member in members:
																			if member.startswith(relative_path.as_posix()					):
																									yield dummy_parent_path.joinpath(lowerCamelCase__					)

							_UpperCamelCase   :						Dict												=  Path(lowerCamelCase__					)
							_UpperCamelCase   :						List[str]												=  _iter_archive_members(lowerCamelCase__					) if self.use_local_dummy_data else path.rglob('*'					)
							for file_path in file_paths:
													if file_path.is_file() and not file_path.name.startswith(('.', '__')					):
																			yield file_path.relative_to(lowerCamelCase__					).as_posix(), file_path.open('rb'					)





	def       UpperCamelCase_    (					self   :							List[str]					,lowerCamelCase__   :							Union[str, Any]					):







							'''simple docstring'''
							if not isinstance(lowerCamelCase__					,lowerCamelCase__					):
													_UpperCamelCase   :						str												=  [paths]
							for path in paths:
													if os.path.isfile(lowerCamelCase__					):
																			if os.path.basename(lowerCamelCase__					).startswith(('.', '__')					):
																									return
																			yield path
													else:
																			for dirpath, dirnames, filenames in os.walk(lowerCamelCase__					):
																									if os.path.basename(lowerCamelCase__					).startswith(('.', '__')					):
																															continue
																									dirnames.sort()
																									for filename in sorted(lowerCamelCase__					):
																															if filename.startswith(('.', '__')					):
																																					continue
																															yield os.path.join(lowerCamelCase__					,lowerCamelCase__					)







'''simple docstring'''
from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

snake_case_					:      Optional[Any]    			=    TypeVar('KEY')
snake_case_					:      Dict    			=    TypeVar('VAL')






@dataclass(frozen=lowercase		,   slots=lowercase  )
class 				lowercase__      (				Generic[KEY, VAL]  ):
	lowercase__    							=     42
	lowercase__    							=     42






class 				lowercase__      (				_Item  ):





	def __init__(					self   :							Union[str, Any]					):







							'''simple docstring'''
							super().__init__(lowerCamelCase__					,lowerCamelCase__					)





	def __bool__(					self   :							Tuple					):







							'''simple docstring'''
							return False
snake_case_					:      int    			=    _DeletedItem()






class 				lowercase__      (				MutableMapping[KEY, VAL]  ):





	def __init__(					self   :							Any					,lowerCamelCase__   :							int = 8					,lowerCamelCase__   :							float = 0.7_5					):







							'''simple docstring'''
							_UpperCamelCase   :						Union[str, Any]												=  initial_block_size
							_UpperCamelCase   :						list[_Item | None]												=  [None] * initial_block_size
							assert 0.0 < capacity_factor < 1.0
							_UpperCamelCase   :						Optional[int]												=  capacity_factor
							_UpperCamelCase   :						List[Any]												=  0





	def       UpperCamelCase_    (					self   :							Optional[int]					,lowerCamelCase__   :							KEY					):







							'''simple docstring'''
							return hash(lowerCamelCase__					) % len(self._buckets					)





	def       UpperCamelCase_    (					self   :							List[Any]					,lowerCamelCase__   :							int					):







							'''simple docstring'''
							return (ind + 1) % len(self._buckets					)





	def       UpperCamelCase_    (					self   :							Union[str, Any]					,lowerCamelCase__   :							int					,lowerCamelCase__   :							KEY					,lowerCamelCase__   :							VAL					):







							'''simple docstring'''
							_UpperCamelCase   :						str												=  self._buckets[ind]
							if not stored:
													_UpperCamelCase   :						str												=  _Item(lowerCamelCase__					,lowerCamelCase__					)
													self._len += 1
													return True
							elif stored.key == key:
													_UpperCamelCase   :						Union[str, Any]												=  _Item(lowerCamelCase__					,lowerCamelCase__					)
													return True
							else:
													return False





	def       UpperCamelCase_    (					self   :							Optional[int]					):







							'''simple docstring'''
							_UpperCamelCase   :						Dict												=  len(self._buckets					) * self._capacity_factor
							return len(self					) >= int(lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							Dict					):







							'''simple docstring'''
							if len(self._buckets					) <= self._initial_block_size:
													return False
							_UpperCamelCase   :						str												=  len(self._buckets					) * self._capacity_factor / 2
							return len(self					) < limit





	def       UpperCamelCase_    (					self   :							int					,lowerCamelCase__   :							int					):







							'''simple docstring'''
							_UpperCamelCase   :						int												=  self._buckets
							_UpperCamelCase   :						Dict												=  [None] * new_size
							_UpperCamelCase   :						str												=  0
							for item in old_buckets:
													if item:
																			self._add_item(item.key					,item.val					)





	def       UpperCamelCase_    (					self   :							int					):







							'''simple docstring'''
							self._resize(len(self._buckets					) * 2					)





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							self._resize(len(self._buckets					) // 2					)





	def       UpperCamelCase_    (					self   :							Union[str, Any]					,lowerCamelCase__   :							KEY					):







							'''simple docstring'''
							_UpperCamelCase   :						List[str]												=  self._get_bucket_index(lowerCamelCase__					)
							for _ in range(len(self._buckets					)					):
													yield ind
													_UpperCamelCase   :						Optional[int]												=  self._get_next_ind(lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							str					,lowerCamelCase__   :							KEY					,lowerCamelCase__   :							VAL					):







							'''simple docstring'''
							for ind in self._iterate_buckets(lowerCamelCase__					):
													if self._try_set(lowerCamelCase__					,lowerCamelCase__					,lowerCamelCase__					):
																			break





	def __setitem__(					self   :							Dict					,lowerCamelCase__   :							KEY					,lowerCamelCase__   :							VAL					):







							'''simple docstring'''
							if self._is_full():
													self._size_up()

							self._add_item(lowerCamelCase__					,lowerCamelCase__					)





	def __delitem__(					self   :							Optional[Any]					,lowerCamelCase__   :							KEY					):







							'''simple docstring'''
							for ind in self._iterate_buckets(lowerCamelCase__					):
													_UpperCamelCase   :						Any												=  self._buckets[ind]
													if item is None:
																			raise KeyError(lowerCamelCase__					)
													if item is _deleted:
																			continue
													if item.key == key:
																			_UpperCamelCase   :						Optional[Any]												=  _deleted
																			self._len -= 1
																			break
							if self._is_sparse():
													self._size_down()





	def __getitem__(					self   :							Dict					,lowerCamelCase__   :							KEY					):







							'''simple docstring'''
							for ind in self._iterate_buckets(lowerCamelCase__					):
													_UpperCamelCase   :						Any												=  self._buckets[ind]
													if item is None:
																			break
													if item is _deleted:
																			continue
													if item.key == key:
																			return item.val
							raise KeyError(lowerCamelCase__					)





	def __len__(					self   :							Dict					):







							'''simple docstring'''
							return self._len





	def __iter__(					self   :							List[str]					):







							'''simple docstring'''
							yield from (item.key for item in self._buckets if item)





	def __repr__(					self   :							Union[str, Any]					):







							'''simple docstring'''
							_UpperCamelCase   :						Tuple												=  ' ,'.join(
							    F'{item.key}: {item.val}' for item in self._buckets if item					)
							return F'HashMap({val_string})'


