

import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase			       =			logging.get_logger(__name__)

_UpperCamelCase			       =			{
    '''source_spm''': '''source.spm''',
    '''target_spm''': '''target.spm''',
    '''vocab''': '''vocab.json''',
    '''target_vocab_file''': '''target_vocab.json''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

_UpperCamelCase			       =			{
    '''source_spm''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm'''
    },
    '''target_spm''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm'''
    },
    '''vocab''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json'''
    },
    '''tokenizer_config_file''': {
        '''Helsinki-NLP/opus-mt-en-de''': (
            '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json'''
        )
    },
}

_UpperCamelCase			       =			{'''Helsinki-NLP/opus-mt-en-de''': 512}
_UpperCamelCase			       =			{}



class     lowercase						(       _UpperCamelCase					):



 '''simple docstring'''


 __SCREAMING_SNAKE_CASE												=				VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE												=				PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE												=				PRETRAINED_INIT_CONFIGURATION
 __SCREAMING_SNAKE_CASE												=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE												=				["""input_ids""", """attention_mask"""]
 __SCREAMING_SNAKE_CASE												=				re.compile(""">>.+<<"""					)  # type: re.Pattern




 def __init__(self       ,       __a       ,       __a       ,       __a       ,       __a=None       ,       __a=None       ,       __a=None       ,       __a="<unk>"       ,       __a="</s>"       ,       __a="<pad>"       ,       __a=512       ,       __a = None       ,       __a=False       ,       **__a       ,       ) ->					None:






  """simple docstring"""





  UpperCAmelCase__							=				{} if sp_model_kwargs is None else sp_model_kwargs

  super().__init__(
      # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
      source_lang=__a       ,       target_lang=__a       ,       unk_token=__a       ,       eos_token=__a       ,       pad_token=__a       ,       model_max_length=__a       ,       sp_model_kwargs=self.sp_model_kwargs       ,       target_vocab_file=__a       ,       separate_vocabs=__a       ,       **__a       ,       )
  assert Path(__a ).exists(), F"cannot find spm source {source_spm}"

  UpperCAmelCase__							=				separate_vocabs
  UpperCAmelCase__							=				load_json(__a )
  if self.unk_token not in self.encoder:
   raise KeyError('<unk> token must be in vocab' )
  assert self.pad_token in self.encoder

  if separate_vocabs:
   UpperCAmelCase__							=				load_json(__a )
   UpperCAmelCase__							=				{v: k for k, v in self.target_encoder.items()}
   UpperCAmelCase__							=				[]
  else:
   UpperCAmelCase__							=				{v: k for k, v in self.encoder.items()}
   UpperCAmelCase__							=				[k for k in self.encoder if k.startswith('>>' ) and k.endswith('<<' )]

  UpperCAmelCase__							=				source_lang
  UpperCAmelCase__							=				target_lang
  UpperCAmelCase__							=				[source_spm, target_spm]

  # load SentencePiece model for pre-processing
  UpperCAmelCase__							=				load_spm(__a       ,       self.sp_model_kwargs )
  UpperCAmelCase__							=				load_spm(__a       ,       self.sp_model_kwargs )
  UpperCAmelCase__							=				self.spm_source
  UpperCAmelCase__							=				self.encoder

  # Multilingual target side: default to using first supported language code.

  self._setup_normalizer()




 def   UpperCamelCase__							(self ) ->					str:






  """simple docstring"""





  try:
   from sacremoses import MosesPunctNormalizer

   UpperCAmelCase__							=				MosesPunctNormalizer(self.source_lang ).normalize
  except (ImportError, FileNotFoundError):
   warnings.warn('Recommended: pip install sacremoses.' )
   UpperCAmelCase__							=				lambda __a : x




 def   UpperCamelCase__							(self       ,       __a ) ->					str:






  """simple docstring"""





  return self.punc_normalizer(__a ) if x else ""




 def   UpperCamelCase__							(self       ,       __a ) ->					List[Any]:






  """simple docstring"""





  return self.current_encoder.get(__a       ,       self.current_encoder[self.unk_token] )




 def   UpperCamelCase__							(self       ,       __a ) ->					Optional[int]:






  """simple docstring"""





  UpperCAmelCase__							=				self.language_code_re.match(__a )
  UpperCAmelCase__							=				[match.group(0 )] if match else []
  return code, self.language_code_re.sub(''       ,       __a )




 def   UpperCamelCase__							(self       ,       __a ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__				,    UpperCAmelCase__							=				self.remove_language_code(__a )
  UpperCAmelCase__							=				self.current_spm.encode(__a       ,       out_type=__a )
  return code + pieces




 def   UpperCamelCase__							(self       ,       __a ) ->					str:






  """simple docstring"""





  return self.decoder.get(__a       ,       self.unk_token )




 def   UpperCamelCase__							(self       ,       __a       ,       **__a ) ->					List[Any]:






  """simple docstring"""





  return super().batch_decode(__a       ,       **__a )




 def   UpperCamelCase__							(self       ,       __a       ,       **__a ) ->					Any:






  """simple docstring"""





  return super().decode(__a       ,       **__a )




 def   UpperCamelCase__							(self       ,       __a ) ->					str:






  """simple docstring"""





  UpperCAmelCase__							=				self.spm_source if self._decode_use_source_tokenizer else self.spm_target
  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				''
  for token in tokens:
   # make sure that special tokens are not decoded using sentencepiece model
   if token in self.all_special_tokens:
    out_string += sp_model.decode_pieces(__a ) + token + " "
    UpperCAmelCase__							=				[]
   else:
    current_sub_tokens.append(__a )
  out_string += sp_model.decode_pieces(__a )
  return out_string.strip()




 def   UpperCamelCase__							(self       ,       __a       ,       __a=None ) ->					List[int]:






  """simple docstring"""





  if token_ids_a is None:
   return token_ids_a + [self.eos_token_id]
  # We don't expect to process pairs, but leave the pair logic for API consistency
  return token_ids_a + token_ids_a + [self.eos_token_id]




 def   UpperCamelCase__							(self ) ->					Optional[int]:






  """simple docstring"""





  UpperCAmelCase__							=				self.spm_source
  UpperCAmelCase__							=				self.encoder




 def   UpperCamelCase__							(self ) ->					List[Any]:






  """simple docstring"""





  UpperCAmelCase__							=				self.spm_target
  if self.separate_vocabs:
   UpperCAmelCase__							=				self.target_encoder




 @property
 def   UpperCamelCase__							(self ) ->					int:






  """simple docstring"""





  return len(self.encoder )




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None ) ->					Tuple[str]:






  """simple docstring"""





  if not os.path.isdir(__a ):
   logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
   return
  UpperCAmelCase__							=				[]

  if self.separate_vocabs:
   UpperCAmelCase__							=				os.path.join(
       __a       ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab']       ,       )
   UpperCAmelCase__							=				os.path.join(
       __a       ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['target_vocab_file']       ,       )
   save_json(self.encoder       ,       __a )
   save_json(self.target_encoder       ,       __a )
   saved_files.append(__a )
   saved_files.append(__a )
  else:
   UpperCAmelCase__							=				os.path.join(
       __a       ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab'] )
   save_json(self.encoder       ,       __a )
   saved_files.append(__a )

  for spm_save_filename, spm_orig_path, spm_model in zip(
      [VOCAB_FILES_NAMES['source_spm'], VOCAB_FILES_NAMES['target_spm']]       ,       self.spm_files       ,       [self.spm_source, self.spm_target]       ,       ):
   UpperCAmelCase__							=				os.path.join(
       __a       ,       (filename_prefix + '-' if filename_prefix else '') + spm_save_filename )
   if os.path.abspath(__a ) != os.path.abspath(__a ) and os.path.isfile(__a ):
    copyfile(__a       ,       __a )
    saved_files.append(__a )
   elif not os.path.isfile(__a ):
    with open(__a       ,       'wb' ) as fi:
     UpperCAmelCase__							=				spm_model.serialized_model_proto()
     fi.write(__a )
    saved_files.append(__a )

  return tuple(__a )




 def   UpperCamelCase__							(self ) ->					Dict:






  """simple docstring"""





  return self.get_src_vocab()




 def   UpperCamelCase__							(self ) ->					Any:






  """simple docstring"""





  return dict(self.encoder       ,       **self.added_tokens_encoder )




 def   UpperCamelCase__							(self ) ->					Dict:






  """simple docstring"""





  return dict(self.target_encoder       ,       **self.added_tokens_decoder )




 def __getstate__(self ) ->					Dict:






  """simple docstring"""





  UpperCAmelCase__							=				self.__dict__.copy()
  state.update(
      {k: None for k in ['spm_source', 'spm_target', 'current_spm', 'punc_normalizer', 'target_vocab_file']} )
  return state




 def __setstate__(self       ,       __a ) ->					None:






  """simple docstring"""





  UpperCAmelCase__							=				d

  # for backward compatibility
  if not hasattr(self       ,       'sp_model_kwargs' ):
   UpperCAmelCase__							=				{}

  UpperCAmelCase__				,    UpperCAmelCase__							=				(load_spm(__a       ,       self.sp_model_kwargs ) for f in self.spm_files)
  UpperCAmelCase__							=				self.spm_source
  self._setup_normalizer()




 def   UpperCamelCase__							(self       ,       *__a       ,       **__a ) ->					Optional[Any]:






  """simple docstring"""





  return 1




 def   UpperCamelCase__							(self       ,       __a ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				set(self.all_special_ids )  # call it once instead of inside list comp
  all_special_ids.remove(self.unk_token_id )  # <unk> is only sometimes special
  return [1 if x in all_special_ids else 0 for x in seq]




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None       ,       __a = False ) ->					List[int]:






  """simple docstring"""





  if already_has_special_tokens:
   return self._special_token_mask(__a )
  elif token_ids_a is None:
   return self._special_token_mask(__a ) + [1]
  else:
   return self._special_token_mask(token_ids_a + token_ids_a ) + [1]







def 						UpperCamelCase_(			snake_case__:  str  ,    snake_case__:  Dict[str, Any]   )       -> sentencepiece.SentencePieceProcessor:
 UpperCAmelCase__							=				sentencepiece.SentencePieceProcessor(**snake_case__   )
 spm.Load(snake_case__   )
 return spm
def 						UpperCamelCase_(			snake_case__:  str  ,    snake_case__:  str   )       -> None:
 with open(snake_case__  ,    'w'   ) as f:
  json.dump(snake_case__  ,    snake_case__  ,    indent=2   )







def 						UpperCamelCase_(			snake_case__:  str   )       -> Union[Dict, List]:
 with open(snake_case__  ,    'r'   ) as f:
  return json.load(snake_case__   )






import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
 import importlib.resources as pkg_resources
except ImportError:
 # Try backported to PY<37 `importlib_resources`.
 import importlib_resources as pkg_resources

from . import resources


_UpperCamelCase			       =			'''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
_UpperCamelCase			       =			F"""{BASE_REF_URL}/{__file__}"""
_UpperCamelCase			       =			logging.getLogger(__name__)
def 						UpperCamelCase_(			snake_case__:  str   )       -> Tuple[Any, str]:
 UpperCAmelCase__							=				pkg_resources.read_text(snake_case__  ,    snake_case__   )
 return yaml.safe_load(snake_case__   ), f"{BASE_REF_URL}/resources/{resource}"


_UpperCamelCase       ,				_UpperCamelCase			       =			load_yaml_resource('''readme_structure.yaml''')

_UpperCamelCase			       =			[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCamelCase			       =			Tuple[dict, List[str], List[str]]



class     lowercase						:



 '''simple docstring'''


 def __init__(self       ,       __a       ,       __a       ,       __a = None       ,       __a = False ) ->					List[str]:






  """simple docstring"""





  UpperCAmelCase__							=				name
  UpperCAmelCase__							=				level
  UpperCAmelCase__							=				lines
  UpperCAmelCase__							=				''
  UpperCAmelCase__							=				True
  UpperCAmelCase__							=				{}
  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if self.lines is not None:
   self.parse(suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self       ,       __a = False ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				''
  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				False
  for line in self.lines:
   if line.strip(' \n' ) == "":
    continue
   elif line.strip(' \n' )[:3] == "```":
    UpperCAmelCase__							=				not code_start
   elif line.split()[0] == self.level + "#" and not code_start:
    if current_sub_level != "":
     UpperCAmelCase__							=				Section(__a       ,       self.level + '#'       ,       __a )
     UpperCAmelCase__							=				[]
    else:
     if current_lines != []:
      self.text += "".join(__a ).strip()
      if self.text != "" and self.text not in FILLER_TEXT:
       UpperCAmelCase__							=				False
      UpperCAmelCase__							=				[]

    UpperCAmelCase__							=				' '.join(line.split()[1:] ).strip(' \n' )
   else:
    current_lines.append(__a )
  else:
   if current_sub_level != "":
    if current_sub_level in self.content:
     self.parsing_error_list.append(
         F"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections." )
    UpperCAmelCase__							=				Section(__a       ,       self.level + '#'       ,       __a )
   else:
    if current_lines != []:
     self.text += "".join(__a ).strip()
     if self.text != "" and self.text not in FILLER_TEXT:
      UpperCAmelCase__							=				False

  if self.level == "" and not suppress_parsing_errors:
   if self.parsing_error_list != [] or self.parsing_warning_list != []:
    UpperCAmelCase__							=				UpperCAmelCase__							=				'\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list )
    UpperCAmelCase__							=				F"The following issues were found while parsing the README at `{self.name}`:\n" + errors
    raise ValueError(__a )




 def   UpperCamelCase__							(self       ,       __a ) ->					ReadmeValidatorOutput:






  """simple docstring"""





  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if structure["allow_empty"] is False:
   # If content is expected
   if self.is_empty_text and self.content == {}:
    # If no content is found, mention it in the error_list
    error_list.append(F"Expected some content in section `{self.name}` but it is empty." )

  if structure["allow_empty_text"] is False:
   # If some text is expected
   if self.is_empty_text:
    # If no text is found, mention it in the error_list
    error_list.append(
        F"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)." )
        # Subsections Validation
  if structure["subsections"] is not None:
   # If subsections are expected
   if self.content == {}:
    # If no subsections are present
    UpperCAmelCase__							=				[subsection['name'] for subsection in structure['subsections']]
    # Mention the expected values in the error_list
    error_list.append(
        F"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values] )}. Found 'None'." )
   else:
    # If some subsections are present
    UpperCAmelCase__							=				[subsection['name'] for subsection in structure['subsections']]
    UpperCAmelCase__							=				False
    for idx, name in enumerate(__a ):
     if name not in self.content:
      # If the expected subsection is not present
      error_list.append(F"Section `{self.name}` is missing subsection: `{name}`." )
      UpperCAmelCase__							=				True
     else:
      # If the subsection is present, validate subsection, return the result
      # and concat the errors from subsection to section error_list

      # Skip sublevel validation if current level is `###`
      if self.level == "###":
       continue
      else:
       UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self.content[name].validate(
           structure['subsections'][idx] )
      error_list += subsec_error_list
      warning_list += subsec_warning_list

    if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
     for name in self.content:
      if name not in structure_names:
       # If an extra subsection is present
       warning_list.append(
           F"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown." )
  if error_list:
   # If there are errors, do not return the dictionary as it is invalid
   return {}, error_list, warning_list
  else:
   return self.to_dict(), error_list, warning_list




 def   UpperCamelCase__							(self ) ->					dict:






  """simple docstring"""





  return {
      "name": self.name,
      "text": self.text,
      "is_empty_text": self.is_empty_text,
      "subsections": [value.to_dict() for value in self.content.values()],
  }







class     lowercase						(       _UpperCamelCase					):  # Level 0



 '''simple docstring'''


 def __init__(self       ,       __a       ,       __a       ,       __a = None       ,       __a = False ) ->					Optional[int]:






  """simple docstring"""





  super().__init__(name=__a       ,       level='' )  # Not using lines here as we need to use a child class parse
  UpperCAmelCase__							=				structure
  UpperCAmelCase__							=				-2
  UpperCAmelCase__							=				0
  UpperCAmelCase__							=				lines
  if self.lines is not None:
   self.parse(suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self ) ->					int:






  """simple docstring"""





  if self.structure is None:
   UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self._validate(__a )
  else:
   UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self._validate(self.structure )
  if error_list != [] or warning_list != []:
   UpperCAmelCase__							=				'\n'.join(['-\t' + x for x in error_list + warning_list] )
   UpperCAmelCase__							=				F"The following issues were found for the README at `{self.name}`:\n" + errors
   raise ValueError(__a )




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a = None       ,       __a = False ) ->					List[str]:






  """simple docstring"""





  with open(__a       ,       encoding='utf-8' ) as f:
   UpperCAmelCase__							=				f.readlines()
  return cls(__a       ,       __a       ,       __a       ,       suppress_parsing_errors=__a )




 @classmethod
 def   UpperCamelCase__							(cls       ,       __a       ,       __a = None       ,       __a = "root"       ,       __a = False ) ->					Dict:






  """simple docstring"""





  UpperCAmelCase__							=				string.split('\n' )
  return cls(__a       ,       __a       ,       __a       ,       suppress_parsing_errors=__a )




 def   UpperCamelCase__							(self       ,       __a = False ) ->					Optional[Any]:






  """simple docstring"""





  UpperCAmelCase__							=				0

  for line in self.lines:
   self.yaml_tags_line_count += 1
   if line.strip(' \n' ) == "---":
    self.tag_count += 1
    if self.tag_count == 2:
     break
   line_count += 1
  if self.tag_count == 2:
   UpperCAmelCase__							=				self.lines[line_count + 1 :]  # Get the last + 1 th item.
  else:
   UpperCAmelCase__							=				self.lines[self.tag_count :]
  super().parse(suppress_parsing_errors=__a )




 def __str__(self ) ->					List[str]:






  """simple docstring"""





  return str(self.to_dict() )




 def   UpperCamelCase__							(self       ,       __a ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				[]
  UpperCAmelCase__							=				[]
  if self.yaml_tags_line_count == 0:
   warning_list.append('Empty YAML markers are present in the README.' )
  elif self.tag_count == 0:
   warning_list.append('No YAML markers are present in the README.' )
  elif self.tag_count == 1:
   warning_list.append('Only the start of YAML tags present in the README.' )
  # Check how many first level sections are present.
  UpperCAmelCase__							=				len(self.content.keys() )
  if num_first_level_keys > 1:
   # If more than one, add to the error list, continue
   error_list.append(
       F"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys() )] )}. Only one heading is expected. Skipping further validation for this README." )
  elif num_first_level_keys < 1:
   # If less than one, append error.
   error_list.append(
       'The README has no first-level headings. One heading is expected. Skipping further validation for this README.' )

  else:
   # If one exactly
   UpperCAmelCase__							=				list(self.content.keys() )[0]  # Get the key
   if start_key.startswith('Dataset Card for' ):  # Check correct start
    # If the starting is correct, validate all the sections
    UpperCAmelCase__				,    UpperCAmelCase__				,    UpperCAmelCase__							=				self.content[start_key].validate(
        readme_structure['subsections'][0] )
    error_list += sec_error_list
    warning_list += sec_warning_list
   else:
    # If not found, append error
    error_list.append(
        'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.' )
  if error_list:
   # If there are errors, do not return the dictionary as it is invalid
   return {}, error_list, warning_list
  else:
   return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
 from argparse import ArgumentParser

 _UpperCamelCase			       =			ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
 ap.add_argument('''readme_filepath''')
 _UpperCamelCase			       =			ap.parse_args()
 _UpperCamelCase			       =			Path(args.readme_filepath)
 _UpperCamelCase			       =			ReadMe.from_readme(readme_filepath)



