



"""simple docstring"""






import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict






@pytest.mark.parametrize(
    '''files''',							[
        ['''full:README.md''', '''dataset_infos.json'''],
        ['''empty:README.md''', '''dataset_infos.json'''],
        ['''dataset_infos.json'''],
        ['''full:README.md'''],
    ],							)
def 			snake_case_    ( A_   :						Dict,							A_   :						List[str]		):


       '''simple docstring'''





       _lowerCamelCase		:							int            =					tmp_path_factory.mktemp('''dset_infos_dir'''		)
       if "full:README.md" in files:
              with open(dataset_infos_dir / '''README.md''',							'''w'''		) as f:
                     f.write('''---\ndataset_info:\n  dataset_size: 42\n---'''		)
       if "empty:README.md" in files:
              with open(dataset_infos_dir / '''README.md''',							'''w'''		) as f:
                     f.write(''''''		)
    # we want to support dataset_infos.json for backward compatibility
       if "dataset_infos.json" in files:
              with open(dataset_infos_dir / '''dataset_infos.json''',							'''w'''		) as f:
                     f.write('''{"default": {"dataset_size": 42}}'''		)
       _lowerCamelCase		:							str            =					DatasetInfosDict.from_directory(A_		)
       assert dataset_infos
       assert dataset_infos["default"].dataset_size == 42






@pytest.mark.parametrize(
    '''dataset_info''',							[
        DatasetInfo(),
        DatasetInfo(
            description='''foo''',							features=Features({'''a''': Value('''int32'''		)}		),							builder_name='''builder''',							config_name='''config''',							version='''1.0.0''',							splits=[{'''name''': '''train'''}],							download_size=42,							),
    ],							)
def 			snake_case_    ( A_   :						str,							A_   :						DatasetInfo		):


       '''simple docstring'''





       _lowerCamelCase		:							Optional[Any]            =					str(A_		)
       dataset_info.write_to_directory(A_		)
       _lowerCamelCase		:							str            =					DatasetInfo.from_directory(A_		)
       assert dataset_info == reloaded
       assert os.path.exists(os.path.join(A_,							'''dataset_info.json'''		)		)






def 			snake_case_    ( ):


       '''simple docstring'''





       _lowerCamelCase		:							Union[str, Any]            =					DatasetInfo(
           description='''foo''',							citation='''bar''',							homepage='''https://foo.bar''',							license='''CC0''',							features=Features({'''a''': Value('''int32'''		)}		),							post_processed={},							supervised_keys=(),							task_templates=[],							builder_name='''builder''',							config_name='''config''',							version='''1.0.0''',							splits=[{'''name''': '''train''', '''num_examples''': 42}],							download_checksums={},							download_size=13_37,							post_processing_size=4_42,							dataset_size=12_34,							size_in_bytes=13_37 + 4_42 + 12_34,							)
       _lowerCamelCase		:							Optional[Any]            =					dataset_info._to_yaml_dict()
       assert sorted(A_		) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML		)
       for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
              assert key in dataset_info_yaml_dict
              assert isinstance(dataset_info_yaml_dict[key],							(list, dict, int, str)		)
       _lowerCamelCase		:							str            =					yaml.safe_dump(A_		)
       _lowerCamelCase		:							Tuple            =					yaml.safe_load(A_		)
       assert dataset_info_yaml_dict == reloaded






def 			snake_case_    ( ):


       '''simple docstring'''





       _lowerCamelCase		:							int            =					DatasetInfo()
       _lowerCamelCase		:							Dict            =					dataset_info._to_yaml_dict()
       assert dataset_info_yaml_dict == {}







@pytest.mark.parametrize(
    '''dataset_infos_dict''',							[
        DatasetInfosDict(),
        DatasetInfosDict({'''default''': DatasetInfo()}		),
        DatasetInfosDict({'''my_config_name''': DatasetInfo()}		),
        DatasetInfosDict(
            {
                '''default''': DatasetInfo(
                    description='''foo''',							features=Features({'''a''': Value('''int32'''		)}		),							builder_name='''builder''',							config_name='''config''',							version='''1.0.0''',							splits=[{'''name''': '''train'''}],							download_size=42,							)
            }		),
        DatasetInfosDict(
            {
                '''v1''': DatasetInfo(dataset_size=42		),
                '''v2''': DatasetInfo(dataset_size=13_37		),
            }		),
    ],							)
def 			snake_case_    ( A_   :						Optional[Any],							A_   :						DatasetInfosDict		):


       '''simple docstring'''





       _lowerCamelCase		:							List[str]            =					str(A_		)
       dataset_infos_dict.write_to_directory(A_		)
       _lowerCamelCase		:							List[Any]            =					DatasetInfosDict.from_directory(A_		)

       # the config_name of the dataset_infos_dict take over the attribute
       for config_name, dataset_info in dataset_infos_dict.items():
              _lowerCamelCase		:							str            =					config_name
              # the yaml representation doesn't include fields like description or citation
              # so we just test that we can recover what we can from the yaml
              _lowerCamelCase		:							Any            =					DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()		)
       assert dataset_infos_dict == reloaded

       if dataset_infos_dict:
              assert os.path.exists(os.path.join(A_,							'''README.md'''		)		)








'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowercase  =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase  =			{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase  =			{
    """google/electra-small-generator""": 5_12,
    """google/electra-base-generator""": 5_12,
    """google/electra-large-generator""": 5_12,
    """google/electra-small-discriminator""": 5_12,
    """google/electra-base-discriminator""": 5_12,
    """google/electra-large-discriminator""": 5_12,
}

_lowercase  =			{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}
class 						a_				(			UpperCAmelCase__						):
	lowercase_       :     Tuple			  =      VOCAB_FILES_NAMES
	lowercase_       :     Optional[int]			  =      PRETRAINED_VOCAB_FILES_MAP
	lowercase_       :     int			  =      PRETRAINED_INIT_CONFIGURATION
	lowercase_       :     str			  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_       :     List[Any]			  =      ElectraTokenizer




	def __init__(       self     :  Optional[Any]       ,		__lowerCAmelCase     :  Optional[Any]=None       ,		__lowerCAmelCase     :  str=None       ,		__lowerCAmelCase     :  Union[str, Any]=True       ,		__lowerCAmelCase     :  int="[UNK]"       ,		__lowerCAmelCase     :  str="[SEP]"       ,		__lowerCAmelCase     :  Optional[Any]="[PAD]"       ,		__lowerCAmelCase     :  Optional[Any]="[CLS]"       ,		__lowerCAmelCase     :  str="[MASK]"       ,		__lowerCAmelCase     :  List[Any]=True       ,		__lowerCAmelCase     :  Optional[int]=None       ,		**__lowerCAmelCase     :  List[Any]       ,		):
							super().__init__(
							    __lowerCAmelCase       ,		tokenizer_file=__lowerCAmelCase       ,		do_lower_case=__lowerCAmelCase       ,		unk_token=__lowerCAmelCase       ,		sep_token=__lowerCAmelCase       ,		pad_token=__lowerCAmelCase       ,		cls_token=__lowerCAmelCase       ,		mask_token=__lowerCAmelCase       ,		tokenize_chinese_chars=__lowerCAmelCase       ,		strip_accents=__lowerCAmelCase       ,		**__lowerCAmelCase       ,		)

							__snake_case						 =   json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
							if (
							    normalizer_state.get('lowercase'       ,		__lowerCAmelCase      ) != do_lower_case
							    or normalizer_state.get('strip_accents'       ,		__lowerCAmelCase      ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'       ,		__lowerCAmelCase      ) != tokenize_chinese_chars
							):
													__snake_case						 =   getattr(__lowerCAmelCase       ,		normalizer_state.pop('type'      )      )
													__snake_case						 =   do_lower_case
													__snake_case						 =   strip_accents
													__snake_case						 =   tokenize_chinese_chars
													__snake_case						 =   normalizer_class(**__lowerCAmelCase      )

							__snake_case						 =   do_lower_case




	def   lowercase__   (       self     :  Union[str, Any]       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  Tuple=None      ):
							__snake_case						 =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output




	def   lowercase__   (       self     :  Any       ,		__lowerCAmelCase     :  List[int]       ,		__lowerCAmelCase     :  Optional[List[int]] = None      ):
							__snake_case						 =   [self.sep_token_id]
							__snake_case						 =   [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep      ) * [0]
							return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




	def   lowercase__   (       self     :  List[Any]       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  Optional[str] = None      ):
							__snake_case						 =   self._tokenizer.model.save(__lowerCAmelCase       ,		name=__lowerCAmelCase      )
							return tuple(__lowerCAmelCase      )

