


import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
				import importlib.resources as pkg_resources
except ImportError:
				# Try backported to PY<37 `importlib_resources`.
				import importlib_resources as pkg_resources

from . import resources


lowerCAmelCase__       :	int     				='https://github.com/huggingface/datasets/tree/main/src/datasets/utils'
lowerCAmelCase__       :	Dict     				=F"""{BASE_REF_URL}/{__file__}"""
lowerCAmelCase__       :	Optional[Any]     				=logging.getLogger(__name__)






def 		a__       (							A__							):
			SCREAMING_SNAKE_CASE_						:			Dict       						= pkg_resources.read_text(SCREAMING_SNAKE_CASE__,	SCREAMING_SNAKE_CASE__							)
			return yaml.safe_load(SCREAMING_SNAKE_CASE__							), F'''{BASE_REF_URL}/resources/{resource}'''


lowerCAmelCase__				,lowerCAmelCase__       :	str     				=load_yaml_resource('readme_structure.yaml')

lowerCAmelCase__       :	Any     				=[
    '[Needs More Information]',
    '[More Information Needed]',
    '(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)',
]

# Dictionary representation of section/readme, error_list, warning_list
lowerCAmelCase__       :	Dict     				=Tuple[dict, List[str], List[str]]







class      __lowercase  :





							"""simple docstring"""







							def __init__(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= name
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= level
										SCREAMING_SNAKE_CASE_						:			List[str]       						= lines
										SCREAMING_SNAKE_CASE_						:			Dict       						= ''''''
										SCREAMING_SNAKE_CASE_						:			Dict       						= True
										SCREAMING_SNAKE_CASE_						:			Dict       						= {}
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= []
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= []
										if self.lines is not None:
													self.parse(suppress_parsing_errors=UpperCamelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			List[Any]       						= ''''''
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= []
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= False
										for line in self.lines:
													if line.strip(' \n'    ) == "":
																continue
													elif line.strip(' \n'    )[:3] == "```":
																SCREAMING_SNAKE_CASE_						:			str       						= not code_start
													elif line.split()[0] == self.level + "#" and not code_start:
																if current_sub_level != "":
																			SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= Section(UpperCamelCase__       ,		self.level + '#'       ,		UpperCamelCase__    )
																			SCREAMING_SNAKE_CASE_						:			str       						= []
																else:
																			if current_lines != []:
																						self.text += "".join(UpperCamelCase__    ).strip()
																						if self.text != "" and self.text not in FILLER_TEXT:
																									SCREAMING_SNAKE_CASE_						:			Tuple       						= False
																						SCREAMING_SNAKE_CASE_						:			Any       						= []

																SCREAMING_SNAKE_CASE_						:			Dict       						= ''' '''.join(line.split()[1:]    ).strip(' \n'    )
													else:
																current_lines.append(UpperCamelCase__    )
										else:
													if current_sub_level != "":
																if current_sub_level in self.content:
																			self.parsing_error_list.append(
																			    F'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''    )
																SCREAMING_SNAKE_CASE_						:			Tuple       						= Section(UpperCamelCase__       ,		self.level + '#'       ,		UpperCamelCase__    )
													else:
																if current_lines != []:
																			self.text += "".join(UpperCamelCase__    ).strip()
																			if self.text != "" and self.text not in FILLER_TEXT:
																						SCREAMING_SNAKE_CASE_						:			List[str]       						= False

										if self.level == "" and not suppress_parsing_errors:
													if self.parsing_error_list != [] or self.parsing_warning_list != []:
																SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= '''\n'''.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list    )
																SCREAMING_SNAKE_CASE_						:			str       						= F'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
																raise ValueError(UpperCamelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			List[Any]       						= []
										SCREAMING_SNAKE_CASE_						:			Tuple       						= []
										if structure["allow_empty"] is False:
													# If content is expected
													if self.is_empty_text and self.content == {}:
																# If no content is found, mention it in the error_list
																error_list.append(F'''Expected some content in section `{self.name}` but it is empty.'''    )

										if structure["allow_empty_text"] is False:
													# If some text is expected
													if self.is_empty_text:
																# If no text is found, mention it in the error_list
																error_list.append(
																    F'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''    )
        # Subsections Validation
										if structure["subsections"] is not None:
													# If subsections are expected
													if self.content == {}:
																# If no subsections are present
																SCREAMING_SNAKE_CASE_						:			List[Any]       						= [subsection['''name'''] for subsection in structure['''subsections''']]
																# Mention the expected values in the error_list
																error_list.append(
																    F'''Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]    )}. Found \'None\'.'''    )
													else:
																# If some subsections are present
																SCREAMING_SNAKE_CASE_						:			Any       						= [subsection['''name'''] for subsection in structure['''subsections''']]
																SCREAMING_SNAKE_CASE_						:			Tuple       						= False
																for idx, name in enumerate(UpperCamelCase__    ):
																			if name not in self.content:
																						# If the expected subsection is not present
																						error_list.append(F'''Section `{self.name}` is missing subsection: `{name}`.'''    )
																						SCREAMING_SNAKE_CASE_						:			int       						= True
																			else:
																						# If the subsection is present, validate subsection, return the result
																						# and concat the errors from subsection to section error_list

																						# Skip sublevel validation if current level is `###`
																						if self.level == "###":
																									continue
																						else:
																									SCREAMING_SNAKE_CASE_						:			Optional[int]       						= self.content[name].validate(
																									    structure['subsections'][idx]    )
																						error_list += subsec_error_list
																						warning_list += subsec_warning_list

																if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																			for name in self.content:
																						if name not in structure_names:
																									# If an extra subsection is present
																									warning_list.append(
																									    F'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''    )
										if error_list:
													# If there are errors, do not return the dictionary as it is invalid
													return {}, error_list, warning_list
										else:
													return self.to_dict(), error_list, warning_list






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										return {
										    "name": self.name,
										    "text": self.text,
										    "is_empty_text": self.is_empty_text,
										    "subsections": [value.to_dict() for value in self.content.values()],
										}







class      __lowercase  (__SCREAMING_SNAKE_CASE   ):  # Level 0





							"""simple docstring"""







							def __init__(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										super().__init__(name=UpperCamelCase__       ,		level=''    )  # Not using lines here as we need to use a child class parse
										SCREAMING_SNAKE_CASE_						:			str       						= structure
										SCREAMING_SNAKE_CASE_						:			Tuple       						= -2
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= 0
										SCREAMING_SNAKE_CASE_						:			int       						= lines
										if self.lines is not None:
													self.parse(suppress_parsing_errors=UpperCamelCase__    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										if self.structure is None:
													SCREAMING_SNAKE_CASE_						:			Any       						= self._validate(UpperCamelCase__    )
										else:
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= self._validate(self.structure    )
										if error_list != [] or warning_list != []:
													SCREAMING_SNAKE_CASE_						:			List[str]       						= '''\n'''.join(['-\t' + x for x in error_list + warning_list]    )
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= F'''The following issues were found for the README at `{self.name}`:\n''' + errors
													raise ValueError(UpperCamelCase__    )






							@classmethod
							def  UpperCamelCase__			(	cls       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										with open(UpperCamelCase__       ,		encoding='utf-8'    ) as f:
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= f.readlines()
										return cls(UpperCamelCase__       ,		UpperCamelCase__       ,		UpperCamelCase__       ,		suppress_parsing_errors=UpperCamelCase__    )






							@classmethod
							def  UpperCamelCase__			(	cls       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = "root"       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			List[Any]       						= string.split('\n'    )
										return cls(UpperCamelCase__       ,		UpperCamelCase__       ,		UpperCamelCase__       ,		suppress_parsing_errors=UpperCamelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= 0

										for line in self.lines:
													self.yaml_tags_line_count += 1
													if line.strip(' \n'    ) == "---":
																self.tag_count += 1
																if self.tag_count == 2:
																			break
													line_count += 1
										if self.tag_count == 2:
													SCREAMING_SNAKE_CASE_						:			str       						= self.lines[line_count + 1 :]  # Get the last + 1 th item.
										else:
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= self.lines[self.tag_count :]
										super().parse(suppress_parsing_errors=UpperCamelCase__    )






							def __str__(	self    ):





										"""simple docstring"""




										return str(self.to_dict()    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= []
										SCREAMING_SNAKE_CASE_						:			int       						= []
										if self.yaml_tags_line_count == 0:
													warning_list.append('Empty YAML markers are present in the README.'    )
										elif self.tag_count == 0:
													warning_list.append('No YAML markers are present in the README.'    )
										elif self.tag_count == 1:
													warning_list.append('Only the start of YAML tags present in the README.'    )
										# Check how many first level sections are present.
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= len(self.content.keys()    )
										if num_first_level_keys > 1:
													# If more than one, add to the error list, continue
													error_list.append(
													    F'''The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()    )]    )}. Only one heading is expected. Skipping further validation for this README.'''    )
										elif num_first_level_keys < 1:
													# If less than one, append error.
													error_list.append(
													    'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'    )

										else:
													# If one exactly
													SCREAMING_SNAKE_CASE_						:			int       						= list(self.content.keys()    )[0]  # Get the key
													if start_key.startswith('Dataset Card for'    ):  # Check correct start
																# If the starting is correct, validate all the sections
																SCREAMING_SNAKE_CASE_						:			Any       						= self.content[start_key].validate(
																    readme_structure['subsections'][0]    )
																error_list += sec_error_list
																warning_list += sec_warning_list
													else:
																# If not found, append error
																error_list.append(
																    'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'    )
										if error_list:
													# If there are errors, do not return the dictionary as it is invalid
													return {}, error_list, warning_list
										else:
													return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
				from argparse import ArgumentParser

				lowerCAmelCase__       :	Optional[Any]     				=ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
				ap.add_argument('readme_filepath')
				lowerCAmelCase__       :	int     				=ap.parse_args()
				lowerCAmelCase__       :	str     				=Path(args.readme_filepath)
				lowerCAmelCase__       :	List[Any]     				=ReadMe.from_readme(readme_filepath)







'''simple docstring'''
#
# Sort large text files in a minimum amount of memory
#
import argparse
import os






class      snake_case__    :







		"""simple docstring"""

		lowerCamelCase							=		"""block_{0}.dat"""



		def __init__(	self							:       List[str]   ,     UpperCamelCase__							:       str	)		->       Union[str, Any]:






				"""simple docstring"""







				snake_case   :    Optional[int]			       =		filename
				snake_case   :    Any			       =		[]



		def 	lowerCAmelCase							(	self							:       Any   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       str	)		->       Dict:






				"""simple docstring"""







				snake_case   :    str			       =		self.BLOCK_FILENAME_FORMAT.format(UpperCamelCase__	)
				with open(UpperCamelCase__   ,     '''w'''	) as file:
						file.write(UpperCamelCase__	)
				self.block_filenames.append(UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       Optional[Any]	)		->       Optional[int]:






				"""simple docstring"""







				return self.block_filenames



		def 	lowerCAmelCase							(	self							:       Optional[int]   ,     UpperCamelCase__							:       Union[str, Any]   ,     UpperCamelCase__							:       Optional[Any]=None	)		->       Dict:






				"""simple docstring"""







				snake_case   :    Optional[Any]			       =		0
				with open(self.filename	) as file:
						while True:
								snake_case   :    List[str]			       =		file.readlines(UpperCamelCase__	)

								if lines == []:
										break

								if sort_key is None:
										lines.sort()
								else:
										lines.sort(key=UpperCamelCase__	)

								self.write_block(''''''.join(UpperCamelCase__	)   ,     UpperCamelCase__	)
								i += 1



		def 	lowerCAmelCase							(	self							:       Dict	)		->       Optional[Any]:






				"""simple docstring"""







				map(os.remove   ,     self.block_filenames	)







class      snake_case__    :







		"""simple docstring"""

		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       Optional[int]	)		->       Dict:






				"""simple docstring"""







				snake_case   :    str			       =		-1
				snake_case   :    Optional[Any]			       =		None

				for i in range(len(UpperCamelCase__	)	):
						if min_str is None or choices[i] < min_str:
								snake_case   :    Union[str, Any]			       =		i

				return min_index







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       str   ,     UpperCamelCase__							:       Tuple	)		->       Dict:






				"""simple docstring"""







				snake_case   :    Dict			       =		files
				snake_case   :    Optional[Any]			       =		set()
				snake_case   :    Tuple			       =		len(UpperCamelCase__	)
				snake_case   :    List[Any]			       =		{i: None for i in range(self.num_buffers	)}



		def 	lowerCAmelCase							(	self							:       Tuple	)		->       int:






				"""simple docstring"""







				return {
				    i: self.buffers[i] for i in range(self.num_buffers	) if i not in self.empty
				}



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       int:






				"""simple docstring"""







				for i in range(self.num_buffers	):
						if self.buffers[i] is None and i not in self.empty:
								snake_case   :    Optional[Any]			       =		self.files[i].readline()

								if self.buffers[i] == "":
										self.empty.add(UpperCamelCase__	)
										self.files[i].close()

				if len(self.empty	) == self.num_buffers:
						return False

				return True



		def 	lowerCAmelCase							(	self							:       Optional[Any]   ,     UpperCamelCase__							:       Any	)		->       str:






				"""simple docstring"""







				snake_case   :    List[Any]			       =		self.buffers[index]
				snake_case   :    Optional[int]			       =		None

				return value







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       List[Any]   ,     UpperCamelCase__							:       Optional[Any]	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    Optional[int]			       =		merge_strategy



		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       Optional[int]   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       Dict	)		->       Tuple:






				"""simple docstring"""







				snake_case   :    List[Any]			       =		FilesArray(self.get_file_handles(UpperCamelCase__   ,     UpperCamelCase__	)	)
				with open(UpperCamelCase__   ,     '''w'''   ,     UpperCamelCase__	) as outfile:
						while buffers.refresh():
								snake_case   :    List[Any]			       =		self.merge_strategy.select(buffers.get_dict()	)
								outfile.write(buffers.unshift(UpperCamelCase__	)	)



		def 	lowerCAmelCase							(	self							:       List[str]   ,     UpperCamelCase__							:       Tuple   ,     UpperCamelCase__							:       str	)		->       int:






				"""simple docstring"""







				snake_case   :    str			       =		{}

				for i in range(len(UpperCamelCase__	)	):
						snake_case   :    Any			       =		open(filenames[i]   ,     '''r'''   ,     UpperCamelCase__	)  # noqa: UP015

				return files







class      snake_case__    :







		"""simple docstring"""

		def __init__(	self							:       Any   ,     UpperCamelCase__							:       Any	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    Optional[Any]			       =		block_size



		def 	lowerCAmelCase							(	self							:       Dict   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       Tuple=None	)		->       Optional[Any]:






				"""simple docstring"""







				snake_case   :    List[str]			       =		self.get_number_blocks(UpperCamelCase__   ,     self.block_size	)
				snake_case   :    Optional[Any]			       =		FileSplitter(UpperCamelCase__	)
				splitter.split(self.block_size   ,     UpperCamelCase__	)

				snake_case   :    Optional[Any]			       =		FileMerger(NWayMerge()	)
				snake_case   :    Any			       =		self.block_size / (num_blocks + 1)
				merger.merge(splitter.get_block_filenames()   ,     filename + '''.out'''   ,     UpperCamelCase__	)

				splitter.cleanup()



		def 	lowerCAmelCase							(	self							:       Optional[Any]   ,     UpperCamelCase__							:       List[str]   ,     UpperCamelCase__							:       Dict	)		->       List[Any]:






				"""simple docstring"""







				return (os.stat(UpperCamelCase__	).st_size / block_size) + 1







def 					_UpperCamelCase		(    SCREAMING_SNAKE_CASE__       )   ->				List[Any]:


		'''simple docstring'''


		if string[-1].lower() == "k":
				return int(string[:-1]       ) * 1024
		elif string[-1].lower() == "m":
				return int(string[:-1]       ) * 1024 * 1024
		elif string[-1].lower() == "g":
				return int(string[:-1]       ) * 1024 * 1024 * 1024
		else:
				return int(SCREAMING_SNAKE_CASE__       )




def 					_UpperCamelCase		(    )   ->				Optional[int]:


		'''simple docstring'''


		snake_case   :    Tuple			       =		argparse.ArgumentParser()
		parser.add_argument(
		    '''-m'''      ,    '''--mem'''      ,    help='''amount of memory to use for sorting'''      ,    default='''100M'''       )
		parser.add_argument(
		    '''filename'''      ,    metavar='''<filename>'''      ,    nargs=1      ,    help='''name of file to sort'''       )
		snake_case   :    Any			       =		parser.parse_args()

		snake_case   :    int			       =		ExternalSort(parse_memory(args.mem       )       )
		sorter.sort(args.filename[0]       )


if __name__ == "__main__":
						main()


