"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


SCREAMING_SNAKE_CASE__      						=  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__      						=  {
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE__      						=  {
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

SCREAMING_SNAKE_CASE__      						=  {
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}



class   lowerCAmelCase_    (			lowerCAmelCase					):



				"""simple docstring"""
				_lowerCAmelCase     :    str			  =						VOCAB_FILES_NAMES
				_lowerCAmelCase     :    str			  =						PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase     :    Union[str, Any]			  =						PRETRAINED_INIT_CONFIGURATION
				_lowerCAmelCase     :    Optional[int]			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase     :    Dict			  =						ElectraTokenizer

				def __init__(				self		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=True		,    lowerCAmelCase="[UNK]"		,    lowerCAmelCase="[SEP]"		,    lowerCAmelCase="[PAD]"		,    lowerCAmelCase="[CLS]"		,    lowerCAmelCase="[MASK]"		,    lowerCAmelCase=True		,    lowerCAmelCase=None		,    **lowerCAmelCase		,    ):






										"""simple docstring"""

										super().__init__(
										    lowerCAmelCase		,    tokenizer_file=lowerCAmelCase		,    do_lower_case=lowerCAmelCase		,    unk_token=lowerCAmelCase		,    sep_token=lowerCAmelCase		,    pad_token=lowerCAmelCase		,    cls_token=lowerCAmelCase		,    mask_token=lowerCAmelCase		,    tokenize_chinese_chars=lowerCAmelCase		,    strip_accents=lowerCAmelCase		,    **lowerCAmelCase		,    )

										snake_case								=			json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
										if (
										    normalizer_state.get('lowercase'		,    lowerCAmelCase    ) != do_lower_case
										    or normalizer_state.get('strip_accents'		,    lowerCAmelCase    ) != strip_accents
										    or normalizer_state.get('handle_chinese_chars'		,    lowerCAmelCase    ) != tokenize_chinese_chars
										):
																snake_case								=			getattr(lowerCAmelCase		,    normalizer_state.pop('type'    )    )
																snake_case								=			do_lower_case
																snake_case								=			strip_accents
																snake_case								=			tokenize_chinese_chars
																snake_case								=			normalizer_class(**lowerCAmelCase    )

										snake_case								=			do_lower_case

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase=None    ):






										"""simple docstring"""

										snake_case								=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






										"""simple docstring"""

										snake_case								=			[self.sep_token_id]
										snake_case								=			[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep    ) * [0]
										return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






										"""simple docstring"""

										snake_case								=			self._tokenizer.model.save(lowerCAmelCase		,    name=lowerCAmelCase    )
										return tuple(lowerCAmelCase    )



"""simple docstring"""
import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)



class   lowerCAmelCase_    (			unittest.TestCase					):



				"""simple docstring"""

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			['a', 'b', 'c']

										# Defaults to last layer if both are None
										snake_case					,snake_case								=			get_aligned_output_features_output_indices(lowerCAmelCase		,    lowerCAmelCase		,    lowerCAmelCase    )
										self.assertEqual(lowerCAmelCase		,    ['c']    )
										self.assertEqual(lowerCAmelCase		,    [2]    )

										# Out indices set to match out features
										snake_case					,snake_case								=			get_aligned_output_features_output_indices(['a', 'c']		,    lowerCAmelCase		,    lowerCAmelCase    )
										self.assertEqual(lowerCAmelCase		,    ['a', 'c']    )
										self.assertEqual(lowerCAmelCase		,    [0, 2]    )

										# Out features set to match out indices
										snake_case					,snake_case								=			get_aligned_output_features_output_indices(lowerCAmelCase		,    [0, 2]		,    lowerCAmelCase    )
										self.assertEqual(lowerCAmelCase		,    ['a', 'c']    )
										self.assertEqual(lowerCAmelCase		,    [0, 2]    )

										# Out features selected from negative indices
										snake_case					,snake_case								=			get_aligned_output_features_output_indices(lowerCAmelCase		,    [-3, -1]		,    lowerCAmelCase    )
										self.assertEqual(lowerCAmelCase		,    ['a', 'c']    )
										self.assertEqual(lowerCAmelCase		,    [-3, -1]    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(['a', 'b']		,    (0, 1)		,    lowerCAmelCase    )

										# Out features must be a list
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(('a', 'b')		,    (0, 1)		,    ['a', 'b']    )

										# Out features must be a subset of stage names
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(['a', 'b']		,    (0, 1)		,    ['a']    )

										# Out indices must be a list or tuple
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(lowerCAmelCase		,    0		,    ['a', 'b']    )

										# Out indices must be a subset of stage names
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(lowerCAmelCase		,    (0, 1)		,    ['a']    )

										# Out features and out indices must be the same length
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(['a', 'b']		,    (0,)		,    ['a', 'b', 'c']    )

										# Out features should match out indices
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(['a', 'b']		,    (0, 2)		,    ['a', 'b', 'c']    )

										# Out features and out indices should be in order
										with self.assertRaises(lowerCAmelCase    ):
																verify_out_features_out_indices(['b', 'a']		,    (0, 1)		,    ['a', 'b']    )

										# Check passes with valid inputs
										verify_out_features_out_indices(['a', 'b', 'd']		,    (0, 1, -1)		,    ['a', 'b', 'c', 'd']    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			BackboneMixin()

										snake_case								=			['a', 'b', 'c']
										snake_case								=			['a', 'c']
										snake_case								=			[0, 2]

										# Check that the output features and indices are set correctly
										self.assertEqual(backbone.out_features		,    ['a', 'c']    )
										self.assertEqual(backbone.out_indices		,    [0, 2]    )

										# Check out features and indices are updated correctly
										snake_case								=			['a', 'b']
										self.assertEqual(backbone.out_features		,    ['a', 'b']    )
										self.assertEqual(backbone.out_indices		,    [0, 1]    )

										snake_case								=			[-3, -1]
										self.assertEqual(backbone.out_features		,    ['a', 'c']    )
										self.assertEqual(backbone.out_indices		,    [-3, -1]    )


