


'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__A	={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A	={
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

__A	={
    'google/electra-small-generator': 5_12,
    'google/electra-base-generator': 5_12,
    'google/electra-large-generator': 5_12,
    'google/electra-small-discriminator': 5_12,
    'google/electra-base-discriminator': 5_12,
    'google/electra-large-discriminator': 5_12,
}

__A	={
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Optional[int]   =					VOCAB_FILES_NAMES
					lowerCAmelCase      :List[Any]   =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase      :Optional[Any]   =					PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase      :Any   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase      :str   =					ElectraTokenizer




					def __init__(      self							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase="[UNK]"							,							_lowerCamelCase="[SEP]"							,							_lowerCamelCase="[PAD]"							,							_lowerCamelCase="[CLS]"							,							_lowerCamelCase="[MASK]"							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							**_lowerCamelCase							,							):
								super().__init__(
								    _lowerCamelCase							,							tokenizer_file=_lowerCamelCase							,							do_lower_case=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							tokenize_chinese_chars=_lowerCamelCase							,							strip_accents=_lowerCamelCase							,							**_lowerCamelCase							,							)

								UpperCAmelCase__  :					Optional[int]        =     json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get("""lowercase"""							,							_lowerCamelCase) != do_lower_case
								    or normalizer_state.get("""strip_accents"""							,							_lowerCamelCase) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars"""							,							_lowerCamelCase) != tokenize_chinese_chars
								):
											UpperCAmelCase__  :					Union[str, Any]        =     getattr(_lowerCamelCase							,							normalizer_state.pop("""type"""))
											UpperCAmelCase__  :					str        =     do_lower_case
											UpperCAmelCase__  :					Dict        =     strip_accents
											UpperCAmelCase__  :					Any        =     tokenize_chinese_chars
											UpperCAmelCase__  :					str        =     normalizer_class(**_lowerCamelCase)

								UpperCAmelCase__  :					List[Any]        =     do_lower_case




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase=None):
								UpperCAmelCase__  :					Dict        =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

								return output




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__  :					int        =     [self.sep_token_id]
								UpperCAmelCase__  :					List[str]        =     [self.cls_token_id]
								if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__  :					List[Any]        =     self._tokenizer.model.save(_lowerCamelCase							,							name=_lowerCamelCase)
								return tuple(_lowerCamelCase)



'''simple docstring'''



import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__A	=logging.get_logger(__name__)







def     _UpperCamelCase      (		UpperCamelCase__					):
			UpperCAmelCase__  :					List[str]        =     R"""\w+[.]\d+"""
			UpperCAmelCase__  :					List[Any]        =     re.findall(UpperCamelCase__ ,      UpperCamelCase__					)
			for pat in pats:
						UpperCAmelCase__  :					str        =     key.replace(UpperCamelCase__ ,      """_""".join(pat.split("""."""					)					)					)
			return key







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					):
			UpperCAmelCase__  :					str        =     pt_tuple_key[:-1] + ("""scale""",)
			if (
			    any("""norm""" in str_ for str_ in pt_tuple_key					)
			    and (pt_tuple_key[-1] == "bias")
			    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
			    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
			):
						UpperCAmelCase__  :					Union[str, Any]        =     pt_tuple_key[:-1] + ("""scale""",)
						return renamed_pt_tuple_key, pt_tensor
			elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
						UpperCAmelCase__  :					str        =     pt_tuple_key[:-1] + ("""scale""",)
						return renamed_pt_tuple_key, pt_tensor

			# embedding
			if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
						UpperCAmelCase__  :					Union[str, Any]        =     pt_tuple_key[:-1] + ("""embedding""",)
						return renamed_pt_tuple_key, pt_tensor

			# conv layer
			UpperCAmelCase__  :					Any        =     pt_tuple_key[:-1] + ("""kernel""",)
			if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
						UpperCAmelCase__  :					Any        =     pt_tensor.transpose(2 ,      3 ,      1 ,      0					)
						return renamed_pt_tuple_key, pt_tensor

			# linear layer
			UpperCAmelCase__  :					Optional[int]        =     pt_tuple_key[:-1] + ("""kernel""",)
			if pt_tuple_key[-1] == "weight":
						UpperCAmelCase__  :					Any        =     pt_tensor.T
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm weight
			UpperCAmelCase__  :					Optional[Any]        =     pt_tuple_key[:-1] + ("""weight""",)
			if pt_tuple_key[-1] == "gamma":
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm bias
			UpperCAmelCase__  :					Dict        =     pt_tuple_key[:-1] + ("""bias""",)
			if pt_tuple_key[-1] == "beta":
						return renamed_pt_tuple_key, pt_tensor

			return pt_tuple_key, pt_tensor







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__=4_2					):
			# Step 1: Convert pytorch tensor to numpy
			UpperCAmelCase__  :					str        =     {k: v.numpy() for k, v in pt_state_dict.items()}

			# Step 2: Since the model is stateless, get random Flax params
			UpperCAmelCase__  :					str        =     flax_model.init_weights(PRNGKey(UpperCamelCase__					)					)

			UpperCAmelCase__  :					Union[str, Any]        =     flatten_dict(UpperCamelCase__					)
			UpperCAmelCase__  :					int        =     {}

			# Need to change some parameters name to match Flax names
			for pt_key, pt_tensor in pt_state_dict.items():
						UpperCAmelCase__  :					str        =     rename_key(UpperCamelCase__					)
						UpperCAmelCase__  :					Optional[Any]        =     tuple(renamed_pt_key.split("""."""					)					)

						# Correctly rename weight parameters
						UpperCAmelCase__   ,       UpperCAmelCase__  :					Tuple        =     rename_key_and_reshape_tensor(UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					)

						if flax_key in random_flax_state_dict:
									if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
												raise ValueError(
												    f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
												    f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''					)

        # also add unexpected weight so that warning is thrown
						UpperCAmelCase__  :					str        =     jnp.asarray(UpperCamelCase__					)

			return unflatten_dict(UpperCamelCase__					)