

import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_UpperCamelCase			       =			logging.getLogger(__name__)



class     lowercase						:



 '''simple docstring'''


 def __init__(self ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				False




 def   UpperCamelCase__							(self       ,       __a       ,       __a       ,       __a       ,       __a ) ->					Tuple:






  """simple docstring"""





  if not self.initialized:
   UpperCAmelCase__							=				RagRetriever(
       __a       ,       question_encoder_tokenizer=__a       ,       generator_tokenizer=__a       ,       index=__a       ,       init_retrieval=__a       ,       )
   UpperCAmelCase__							=				True




 def   UpperCamelCase__							(self ) ->					Union[str, Any]:






  """simple docstring"""





  self.retriever.index.init_index()




 def   UpperCamelCase__							(self ) ->					Union[str, Any]:






  """simple docstring"""





  del self.retriever
  UpperCAmelCase__							=				False




 def   UpperCamelCase__							(self       ,       __a       ,       __a ) ->					int:






  """simple docstring"""





  UpperCAmelCase__				,    UpperCAmelCase__							=				self.retriever._main_retrieve(__a       ,       __a )
  UpperCAmelCase__							=				self.retriever.index.get_doc_dicts(__a )
  return doc_ids, retrieved_doc_embeds, doc_dicts







class     lowercase						(       _UpperCamelCase					):



 '''simple docstring'''


 def __init__(self       ,       __a       ,       __a       ,       __a       ,       __a       ,       __a=None ) ->					List[Any]:






  """simple docstring"""





  if index is not None and index.is_initialized() and len(__a ) > 0:
   raise ValueError(
       'When using Ray for distributed fine-tuning, '
       'you\'ll need to provide the paths instead, '
       'as the dataset and the index are loaded '
       'separately. More info in examples/rag/use_own_knowledge_dataset.py ' )

  super().__init__(
      __a       ,       question_encoder_tokenizer=__a       ,       generator_tokenizer=__a       ,       index=__a       ,       init_retrieval=__a       ,       )

  UpperCAmelCase__							=				retrieval_workers
  UpperCAmelCase__							=				question_encoder_tokenizer
  UpperCAmelCase__							=				generator_tokenizer
  if len(self.retrieval_workers ) > 0:
   ray.get(
       [
           worker.create_rag_retriever.remote(__a       ,       __a       ,       __a       ,       __a )
           for worker in self.retrieval_workers
       ] )




 def   UpperCamelCase__							(self ) ->					Optional[Any]:






  """simple docstring"""





  logger.info('initializing retrieval' )

  if len(self.retrieval_workers ) > 0:
   ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers] )
  else:
   # Non-distributed training. Load index into this same process.
   self.index.init_index()




 def   UpperCamelCase__							(self       ,       __a       ,       __a ) ->					Tuple:






  """simple docstring"""





  if len(self.retrieval_workers ) > 0:
   # Select a random retrieval actor.
   UpperCAmelCase__							=				self.retrieval_workers[random.randint(0       ,       len(self.retrieval_workers ) - 1 )]
   UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				ray.get(
       random_worker.retrieve.remote(__a       ,       __a ) )
  else:
   UpperCAmelCase__				,    UpperCAmelCase__							=				self._main_retrieve(__a       ,       __a )
   UpperCAmelCase__							=				self.index.get_doc_dicts(__a )
  return retrieved_doc_embeds, doc_ids, doc_dicts




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a=None       ,       **__a ) ->					Optional[Any]:






  """simple docstring"""





  return super(__a       ,       cls ).get_tokenizers(__a       ,       __a       ,       **__a )




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a       ,       __a=None       ,       **__a ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				kwargs.pop('config'       ,       __a ) or RagConfig.from_pretrained(__a       ,       **__a )
  UpperCAmelCase__							=				RagTokenizer.from_pretrained(__a       ,       config=__a )
  UpperCAmelCase__							=				rag_tokenizer.question_encoder
  UpperCAmelCase__							=				rag_tokenizer.generator

  if indexed_dataset is not None:
   UpperCAmelCase__							=				'custom'
   UpperCAmelCase__							=				CustomHFIndex(config.retrieval_vector_size       ,       __a )
  else:
   UpperCAmelCase__							=				cls._build_index(__a )

  return cls(
      __a       ,       question_encoder_tokenizer=__a       ,       generator_tokenizer=__a       ,       retrieval_workers=__a       ,       index=__a       ,       )




 def   UpperCamelCase__							(self ) ->					List[str]:






  """simple docstring"""





  logger.info('re-loading the new dataset with embeddings' )
  # access from the training loop

  ray.get([worker.clear_object.remote() for worker in self.retrieval_workers] )

  # build the index object again
  UpperCAmelCase__							=				self._build_index(self.config )

  ray.get(
      [
          worker.create_rag_retriever.remote(
              self.config       ,       self.question_encoder_tokenizer       ,       self.generator_tokenizer       ,       __a )
          for worker in self.retrieval_workers
      ] )






import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
 import importlib.resources as pkg_resources
except ImportError:
 # Try backported to PY<37 `importlib_resources`.
 import importlib_resources as pkg_resources

from . import resources


_UpperCamelCase			       =			'''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
_UpperCamelCase			       =			F"""{BASE_REF_URL}/{__file__}"""
_UpperCamelCase			       =			logging.getLogger(__name__)
def 						UpperCamelCase_(			snake_case__:  str   )       -> Tuple[Any, str]:
 UpperCAmelCase__							=				pkg_resources.read_text(snake_case__  ,    snake_case__   )
 return yaml.safe_load(snake_case__   ), f"{BASE_REF_URL}/resources/{resource}"


_UpperCamelCase       ,				_UpperCamelCase			       =			load_yaml_resource('''readme_structure.yaml''')

_UpperCamelCase			       =			[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCamelCase			       =			Tuple[dict, List[str], List[str]]



class     lowercase						:



 '''simple docstring'''


 def __init__(self       ,       __a       ,       __a       ,       __a = None       ,       __a = False ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				name
  UpperCAmelCase__							=				level
  UpperCAmelCase__							=				lines
  UpperCAmelCase__							=				''
  UpperCAmelCase__							=				True
  UpperCAmelCase__							=				{}
  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if self.lines is not None:
   self.parse(suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self       ,       __a = False ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				''
  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				False
  for line in self.lines:
   if line.strip(' \n' ) == "":
    continue
   elif line.strip(' \n' )[:3] == "```":
    UpperCAmelCase__							=				not code_start
   elif line.split()[0] == self.level + "#" and not code_start:
    if current_sub_level != "":
     UpperCAmelCase__							=				Section(__a       ,       self.level + '#'       ,       __a )
     UpperCAmelCase__							=				[]
    else:
     if current_lines != []:
      self.text += "".join(__a ).strip()
      if self.text != "" and self.text not in FILLER_TEXT:
       UpperCAmelCase__							=				False
      UpperCAmelCase__							=				[]

    UpperCAmelCase__							=				' '.join(line.split()[1:] ).strip(' \n' )
   else:
    current_lines.append(__a )
  else:
   if current_sub_level != "":
    if current_sub_level in self.content:
     self.parsing_error_list.append(
         F"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections." )
    UpperCAmelCase__							=				Section(__a       ,       self.level + '#'       ,       __a )
   else:
    if current_lines != []:
     self.text += "".join(__a ).strip()
     if self.text != "" and self.text not in FILLER_TEXT:
      UpperCAmelCase__							=				False

  if self.level == "" and not suppress_parsing_errors:
   if self.parsing_error_list != [] or self.parsing_warning_list != []:
    UpperCAmelCase__							=				UpperCAmelCase__							=				'\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list )
    UpperCAmelCase__							=				F"The following issues were found while parsing the README at `{self.name}`:\n" + errors
    raise ValueError(__a )




 def   UpperCamelCase__							(self       ,       __a ) ->					ReadmeValidatorOutput:






  """simple docstring"""





  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if structure["allow_empty"] is False:
   # If content is expected
   if self.is_empty_text and self.content == {}:
    # If no content is found, mention it in the error_list
    error_list.append(F"Expected some content in section `{self.name}` but it is empty." )

  if structure["allow_empty_text"] is False:
   # If some text is expected
   if self.is_empty_text:
    # If no text is found, mention it in the error_list
    error_list.append(
        F"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)." )
        # Subsections Validation
  if structure["subsections"] is not None:
   # If subsections are expected
   if self.content == {}:
    # If no subsections are present
    UpperCAmelCase__							=				[subsection['name'] for subsection in structure['subsections']]
    # Mention the expected values in the error_list
    error_list.append(
        F"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values] )}. Found 'None'." )
   else:
    # If some subsections are present
    UpperCAmelCase__							=				[subsection['name'] for subsection in structure['subsections']]
    UpperCAmelCase__							=				False
    for idx, name in enumerate(__a ):
     if name not in self.content:
      # If the expected subsection is not present
      error_list.append(F"Section `{self.name}` is missing subsection: `{name}`." )
      UpperCAmelCase__							=				True
     else:
      # If the subsection is present, validate subsection, return the result
      # and concat the errors from subsection to section error_list

      # Skip sublevel validation if current level is `###`
      if self.level == "###":
       continue
      else:
       UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self.content[name].validate(
           structure['subsections'][idx] )
      error_list += subsec_error_list
      warning_list += subsec_warning_list

    if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
     for name in self.content:
      if name not in structure_names:
       # If an extra subsection is present
       warning_list.append(
           F"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown." )
  if error_list:
   # If there are errors, do not return the dictionary as it is invalid
   return {}, error_list, warning_list
  else:
   return self.to_dict(), error_list, warning_list




 def   UpperCamelCase__							(self ) ->					dict:






  """simple docstring"""





  return {
      "name": self.name,
      "text": self.text,
      "is_empty_text": self.is_empty_text,
      "subsections": [value.to_dict() for value in self.content.values()],
  }







class     lowercase						(       _UpperCamelCase					):  # Level 0



 '''simple docstring'''


 def __init__(self       ,       __a       ,       __a       ,       __a = None       ,       __a = False ) ->					Optional[int]:






  """simple docstring"""





  super().__init__(name=__a       ,       level='' )  # Not using lines here as we need to use a child class parse
  UpperCAmelCase__							=				structure
  UpperCAmelCase__							=				-2
  UpperCAmelCase__							=				0
  UpperCAmelCase__							=				lines
  if self.lines is not None:
   self.parse(suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self ) ->					int:






  """simple docstring"""





  if self.structure is None:
   UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self._validate(__a )
  else:
   UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self._validate(self.structure )
  if error_list != [] or warning_list != []:
   UpperCAmelCase__							=				'\n'.join(['-\t' + x for x in error_list + warning_list] )
   UpperCAmelCase__							=				F"The following issues were found for the README at `{self.name}`:\n" + errors
   raise ValueError(__a )




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a = None       ,       __a = False ) ->					List[str]:






  """simple docstring"""





  with open(__a       ,       encoding='utf-8' ) as f:
   UpperCAmelCase__							=				f.readlines()
  return cls(__a       ,       __a       ,       __a       ,       suppress_parsing_errors=__a )




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a = None       ,       __a = "root"       ,       __a = False ) ->					Dict:






  """simple docstring"""





  UpperCAmelCase__							=				string.split('\n' )
  return cls(__a       ,       __a       ,       __a       ,       suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self       ,       __a = False ) ->					Optional[Any]:






  """simple docstring"""





  UpperCAmelCase__							=				0

  for line in self.lines:
   self.yaml_tags_line_count += 1
   if line.strip(' \n' ) == "---":
    self.tag_count += 1
    if self.tag_count == 2:
     break
   line_count += 1
  if self.tag_count == 2:
   UpperCAmelCase__							=				self.lines[line_count + 1 :]  # Get the last + 1 th item.
  else:
   UpperCAmelCase__							=				self.lines[self.tag_count :]
  super().parse(suppress_parsing_errors=__a )




 def __str__(self ) ->					List[str]:






  """simple docstring"""





  return str(self.to_dict() )




 def   UpperCamelCase__							(self       ,       __a ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if self.yaml_tags_line_count == 0:
   warning_list.append('Empty YAML markers are present in the README.' )
  elif self.tag_count == 0:
   warning_list.append('No YAML markers are present in the README.' )
  elif self.tag_count == 1:
   warning_list.append('Only the start of YAML tags present in the README.' )
  # Check how many first level sections are present.
  UpperCAmelCase__							=				len(self.content.keys() )
  if num_first_level_keys > 1:
   # If more than one, add to the error list, continue
   error_list.append(
       F"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys() )] )}. Only one heading is expected. Skipping further validation for this README." )
  elif num_first_level_keys < 1:
   # If less than one, append error.
   error_list.append(
       'The README has no first-level headings. One heading is expected. Skipping further validation for this README.' )

  else:
   # If one exactly
   UpperCAmelCase__							=				list(self.content.keys() )[0]  # Get the key
   if start_key.startswith('Dataset Card for' ):  # Check correct start
    # If the starting is correct, validate all the sections
    UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self.content[start_key].validate(
        readme_structure['subsections'][0] )
    error_list += sec_error_list
    warning_list += sec_warning_list
   else:
    # If not found, append error
    error_list.append(
        'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.' )
  if error_list:
   # If there are errors, do not return the dictionary as it is invalid
   return {}, error_list, warning_list
  else:
   return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
 from argparse import ArgumentParser

 _UpperCamelCase			       =			ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
 ap.add_argument('''readme_filepath''')
 _UpperCamelCase			       =			ap.parse_args()
 _UpperCamelCase			       =			Path(args.readme_filepath)
 _UpperCamelCase			       =			ReadMe.from_readme(readme_filepath)



