



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__snake_case     :int			       ={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case     :int			       ={
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

__snake_case     :Tuple			       ={
    'google/electra-small-generator': 512,
    'google/electra-base-generator': 512,
    'google/electra-large-generator': 512,
    'google/electra-small-discriminator': 512,
    'google/electra-base-discriminator': 512,
    'google/electra-large-discriminator': 512,
}

__snake_case     :Union[str, Any]			       ={
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}



class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     Union[str, Any]								=     VOCAB_FILES_NAMES
				A_   :     Optional[Any]								=     PRETRAINED_VOCAB_FILES_MAP
				A_   :     Optional[int]								=     PRETRAINED_INIT_CONFIGURATION
				A_   :     Tuple								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A_   :     Dict								=     ElectraTokenizer


				def __init__(							self      :  List[Any]						, __UpperCamelCase      :  Union[str, Any]=None						, __UpperCamelCase      :  Union[str, Any]=None						, __UpperCamelCase      :  Tuple=True						, __UpperCamelCase      :  Optional[int]="[UNK]"						, __UpperCamelCase      :  Any="[SEP]"						, __UpperCamelCase      :  Optional[Any]="[PAD]"						, __UpperCamelCase      :  Union[str, Any]="[CLS]"						, __UpperCamelCase      :  Optional[Any]="[MASK]"						, __UpperCamelCase      :  Optional[int]=True						, __UpperCamelCase      :  Optional[Any]=None						, **__UpperCamelCase      :  Dict						, )		->     int:
											super().__init__(
											    __UpperCamelCase						, tokenizer_file=__UpperCamelCase						, do_lower_case=__UpperCamelCase						, unk_token=__UpperCamelCase						, sep_token=__UpperCamelCase						, pad_token=__UpperCamelCase						, cls_token=__UpperCamelCase						, mask_token=__UpperCamelCase						, tokenize_chinese_chars=__UpperCamelCase						, strip_accents=__UpperCamelCase						, **__UpperCamelCase						, )

											A      						=    json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
											if (
											    normalizer_state.get('lowercase'						, __UpperCamelCase       ) != do_lower_case
											    or normalizer_state.get('strip_accents'						, __UpperCamelCase       ) != strip_accents
											    or normalizer_state.get('handle_chinese_chars'						, __UpperCamelCase       ) != tokenize_chinese_chars
											):
																		A      						=    getattr(__UpperCamelCase						, normalizer_state.pop('type'       )       )
																		A      						=    do_lower_case
																		A      						=    strip_accents
																		A      						=    tokenize_chinese_chars
																		A      						=    normalizer_class(**__UpperCamelCase       )

											A      						=    do_lower_case


				def __UpperCamelCase						(							self      :  Tuple						, __UpperCamelCase      :  int						, __UpperCamelCase      :  Union[str, Any]=None       )		->     int:
											A      						=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  List[int]						, __UpperCamelCase      :  Optional[List[int]] = None       )		->     List[int]:
											A      						=    [self.sep_token_id]
											A      						=    [self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[str] = None       )		->     Tuple[str]:
											A      						=    self._tokenizer.model.save(__UpperCamelCase						, name=__UpperCamelCase       )
											return tuple(__UpperCamelCase       )




from typing import List

import numpy as np






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict				)       ->							int:

							'''simple docstring'''






							A      						=    {key: len(lowerCAmelCase__				) for key, value in gen_kwargs.items() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							if len(set(lists_lengths.values()				)				) > 1:
														raise RuntimeError(
														    (
														        'Sharding is ambiguous for this dataset: '
														        + 'we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'
														        + '\n'.join(F'''\t- key {key} has length {length}''' for key, length in lists_lengths.items()				)
														        + '\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '
														        + 'and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'
														    )				)
							A      						=    max(lists_lengths.values()			,	default=0				)
							return max(1			,	lowerCAmelCase__				)






def  lowerCamelCase_	(      lowerCAmelCase__	:	int			,	lowerCAmelCase__	:	int				)       ->							List[range]:

							'''simple docstring'''






							A      						=    []
							for group_idx in range(lowerCAmelCase__				):
														A      						=    num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
														if num_shards_to_add == 0:
																					break
														A      						=    shards_indices_per_group[-1].stop if shards_indices_per_group else 0
														A      						=    range(lowerCAmelCase__			,	start + num_shards_to_add				)
														shards_indices_per_group.append(lowerCAmelCase__				)
							return shards_indices_per_group






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict			,	lowerCAmelCase__	:	int				)       ->							List[dict]:

							'''simple docstring'''






							A      						=    _number_of_shards_in_gen_kwargs(lowerCAmelCase__				)
							if num_shards == 1:
														return [dict(lowerCAmelCase__				)]
							else:
														A      						=    _distribute_shards(num_shards=lowerCAmelCase__			,	max_num_jobs=lowerCAmelCase__				)
														return [
														    {
														        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
														        if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)
														        else value
														        for key, value in gen_kwargs.items()
														    }
														    for group_idx in range(len(lowerCAmelCase__				)				)
														]






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[dict]				)       ->							dict:

							'''simple docstring'''






							return {
							    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
							    if isinstance(gen_kwargs_list[0][key]			,	lowerCAmelCase__				)
							    else gen_kwargs_list[0][key]
							    for key in gen_kwargs_list[0]
							}






def  lowerCamelCase_	(      lowerCAmelCase__	:	np.random.Generator			,	lowerCAmelCase__	:	dict				)       ->							dict:

							'''simple docstring'''






							A      						=    {len(lowerCAmelCase__				) for value in gen_kwargs.values() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							A      						=    {}
							for size in list_sizes:
														A      						=    list(range(lowerCAmelCase__				)				)
														rng.shuffle(indices_per_size[size]				)
							# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
							A      						=    dict(lowerCAmelCase__				)
							for key, value in shuffled_kwargs.items():
														if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				):
																					A      						=    [value[i] for i in indices_per_size[len(lowerCAmelCase__				)]]
							return shuffled_kwargs