


'''simple docstring'''






# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


__UpperCAmelCase					       =							get_logger(__name__)




class 	a__       :





							'''simple docstring'''






							lowercase__    :     List[str]					=   "dummy_data"
							lowercase__    :     Dict					=   "datasets"
							lowercase__    :     Any					=   False







							def __init__(	self      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_ = None      ,					lowerCamelCase_ = False      ,					lowerCamelCase_ = True      ,					lowerCamelCase_ = None      ,					) ->							int:
														lowerCAmelCase__      =			0
														lowerCAmelCase__      =			dataset_name
														lowerCAmelCase__      =			cache_dir
														lowerCAmelCase__      =			use_local_dummy_data
														lowerCAmelCase__      =			config
														# download_callbacks take a single url as input
														lowerCAmelCase__      =			download_callbacks or []
														# if False, it doesn't load existing files and it returns the paths of the dummy files relative
														# to the dummy_data zip file root
														lowerCAmelCase__      =			load_existing_dummy_data

														# TODO(PVP, QL) might need to make this more general
														lowerCAmelCase__      =			str(__A     )
														# to be downloaded
														lowerCAmelCase__      =			None
														lowerCAmelCase__      =			None







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														if self._dummy_file is None:
																					lowerCAmelCase__      =			self.download_dummy_data()
														return self._dummy_file







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Any:
														if self.config is not None:
																					# structure is dummy / config_name / version_name
																					return os.path.join('''dummy'''      ,					self.config.name      ,					self.version_name     )
														# structure is dummy / version_name
														return os.path.join('''dummy'''      ,					self.version_name     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														return os.path.join(self.dummy_data_folder      ,					'''dummy_data.zip'''     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														lowerCAmelCase__      =			(
														    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
														)

														lowerCAmelCase__      =			cached_path(
														    __A      ,					cache_dir=self.cache_dir      ,					extract_compressed_file=__A      ,					force_extract=__A     )

														return os.path.join(__A      ,					self.dummy_file_name     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														return os.path.join(self.datasets_scripts_dir      ,					self.dataset_name      ,					self.dummy_zip_file     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														if self._bucket_url is None:
																					lowerCAmelCase__      =			hf_github_url(self.dataset_name      ,					self.dummy_zip_file.replace(os.sep      ,					'''/'''     )     )
														return self._bucket_url







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														# return full path if its a dir
														if os.path.isdir(self.dummy_file     ):
																					return self.dummy_file
														# else cut off path to file -> example `xsum`.
														return "/".join(self.dummy_file.replace(os.sep      ,					'''/'''     ).split('''/'''     )[:-1]     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					*lowerCamelCase_     ) ->							Optional[int]:
														if self.load_existing_dummy_data:
																					# dummy data is downloaded and tested
																					lowerCAmelCase__      =			self.dummy_file
														else:
																					# dummy data cannot be downloaded and only the path to dummy file is returned
																					lowerCAmelCase__      =			self.dummy_file_name

														# special case when data_url is a dict
														if isinstance(__A      ,					__A     ):
																					return self.create_dummy_data_dict(__A      ,					__A     )
														elif isinstance(__A      ,					(list, tuple)     ):
																					return self.create_dummy_data_list(__A      ,					__A     )
														else:
																					return self.create_dummy_data_single(__A      ,					__A     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					*lowerCamelCase_     ) ->							Optional[Any]:
														return self.download_and_extract(__A     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							int:
														return self.download_and_extract(__A     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							Union[str, Any]:
														return path







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Tuple:
														return {}







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							Dict:
														lowerCAmelCase__      =			{}
														for key, single_urls in data_url.items():
																					for download_callback in self.download_callbacks:
																												if isinstance(__A      ,					__A     ):
																																			for single_url in single_urls:
																																										download_callback(__A     )
																												else:
																																			lowerCAmelCase__      =			single_urls
																																			download_callback(__A     )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
																					if isinstance(__A      ,					__A     ):
																												lowerCAmelCase__      =			[os.path.join(__A      ,					urllib.parse.quote_plus(Path(__A     ).name     )     ) for x in single_urls]
																					else:
																												lowerCAmelCase__      =			single_urls
																												lowerCAmelCase__      =			os.path.join(__A      ,					urllib.parse.quote_plus(Path(__A     ).name     )     )
																					lowerCAmelCase__      =			value

														# make sure that values are unique
														if all(isinstance(__A      ,					__A     ) for i in dummy_data_dict.values()     ) and len(set(dummy_data_dict.values()     )     ) < len(
														    dummy_data_dict.values()     ):
																					# append key to value to make its name unique
																					lowerCAmelCase__      =			{key: value + key for key, value in dummy_data_dict.items()}

														return dummy_data_dict







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							str:
														lowerCAmelCase__      =			[]
														# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
														lowerCAmelCase__      =			all(bool(re.findall('''[0-9]{3,}-of-[0-9]{3,}'''      ,					__A     )     ) for url in data_url     )
														lowerCAmelCase__      =			all(
														    url.startswith('''https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed'''     ) for url in data_url     )
														if data_url and (is_tf_records or is_pubmed_records):
																					lowerCAmelCase__      =			[data_url[0]] * len(__A     )
														for single_url in data_url:
																					for download_callback in self.download_callbacks:
																												download_callback(__A     )
																					# we force the name of each key to be the last file / folder name of the url path
																					# if the url has arguments, we need to encode them with urllib.parse.quote_plus
																					lowerCAmelCase__      =			os.path.join(__A      ,					urllib.parse.quote_plus(single_url.split('''/'''     )[-1]     )     )
																					dummy_data_list.append(__A     )
														return dummy_data_list







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							Optional[int]:
														for download_callback in self.download_callbacks:
																					download_callback(__A     )
														# we force the name of each key to be the last file / folder name of the url path
														# if the url has arguments, we need to encode them with urllib.parse.quote_plus
														lowerCAmelCase__      =			os.path.join(__A      ,					urllib.parse.quote_plus(data_url.split('''/'''     )[-1]     )     )
														if os.path.exists(__A     ) or not self.load_existing_dummy_data:
																					return value
														else:
																					# Backward compatibility, maybe deprecate at one point.
																					# For many datasets with single url calls to dl_manager.download_and_extract,
																					# the dummy_data.zip file is actually the zipped downloaded file
																					# while now we expected the dummy_data.zip file to be a directory containing
																					# the downloaded file.
																					return path_to_dummy_data







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Any:
														pass







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union[str, Any]:
														pass







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Tuple:
														def _iter_archive_members(lowerCamelCase_     ):
																					# this preserves the order of the members inside the ZIP archive
																					lowerCAmelCase__      =			Path(self.dummy_file     ).parent
																					lowerCAmelCase__      =			path.relative_to(__A     )
																					with ZipFile(self.local_path_to_dummy_data     ) as zip_file:
																												lowerCAmelCase__      =			zip_file.namelist()
																					for member in members:
																												if member.startswith(relative_path.as_posix()     ):
																																			yield dummy_parent_path.joinpath(__A     )

														lowerCAmelCase__      =			Path(__A     )
														lowerCAmelCase__      =			_iter_archive_members(__A     ) if self.use_local_dummy_data else path.rglob('''*'''     )
														for file_path in file_paths:
																					if file_path.is_file() and not file_path.name.startswith(('''.''', '''__''')     ):
																												yield file_path.relative_to(__A     ).as_posix(), file_path.open('''rb'''     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							str:
														if not isinstance(__A      ,					__A     ):
																					lowerCAmelCase__      =			[paths]
														for path in paths:
																					if os.path.isfile(__A     ):
																												if os.path.basename(__A     ).startswith(('''.''', '''__''')     ):
																																			return
																												yield path
																					else:
																												for dirpath, dirnames, filenames in os.walk(__A     ):
																																			if os.path.basename(__A     ).startswith(('''.''', '''__''')     ):
																																										continue
																																			dirnames.sort()
																																			for filename in sorted(__A     ):
																																										if filename.startswith(('''.''', '''__''')     ):
																																																	continue
																																										yield os.path.join(__A      ,					__A     )



'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Dict					=   "ctrl"
							lowercase__    :     str					=   ["past_key_values"]
							lowercase__    :     Optional[int]					=   {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}







							def __init__(	self      ,					lowerCamelCase_=24_65_34      ,					lowerCamelCase_=2_56      ,					lowerCamelCase_=12_80      ,					lowerCamelCase_=81_92      ,					lowerCamelCase_=48      ,					lowerCamelCase_=16      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=1e-6      ,					lowerCamelCase_=0.02      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Union[str, Any]:
														lowerCAmelCase__      =			vocab_size
														lowerCAmelCase__      =			n_positions
														lowerCAmelCase__      =			n_embd
														lowerCAmelCase__      =			n_layer
														lowerCAmelCase__      =			n_head
														lowerCAmelCase__      =			dff
														lowerCAmelCase__      =			resid_pdrop
														lowerCAmelCase__      =			embd_pdrop
														lowerCAmelCase__      =			layer_norm_epsilon
														lowerCAmelCase__      =			initializer_range

														lowerCAmelCase__      =			use_cache

														super().__init__(**lowerCamelCase_     )