






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


SCREAMING_SNAKE_CASE__		:							List[str] 				=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__		:							Union[str, Any] 				=				{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__		:							List[str] 				=				{
    """google/electra-small-generator""": 5_12,
    """google/electra-base-generator""": 5_12,
    """google/electra-large-generator""": 5_12,
    """google/electra-small-discriminator""": 5_12,
    """google/electra-base-discriminator""": 5_12,
    """google/electra-large-discriminator""": 5_12,
}

SCREAMING_SNAKE_CASE__		:							int 				=				{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     __lowerCamelCase      				=  VOCAB_FILES_NAMES
     __lowerCamelCase      				=  PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase      				=  PRETRAINED_INIT_CONFIGURATION
     __lowerCamelCase      				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase      				=  ElectraTokenizer




     def __init__(				self	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=True	,						_lowerCAmelCase="[UNK]"	,						_lowerCAmelCase="[SEP]"	,						_lowerCAmelCase="[PAD]"	,						_lowerCAmelCase="[CLS]"	,						_lowerCAmelCase="[MASK]"	,						_lowerCAmelCase=True	,						_lowerCAmelCase=None	,						**_lowerCAmelCase	,						):
          super().__init__(
              _lowerCAmelCase	,						tokenizer_file=_lowerCAmelCase	,						do_lower_case=_lowerCAmelCase	,						unk_token=_lowerCAmelCase	,						sep_token=_lowerCAmelCase	,						pad_token=_lowerCAmelCase	,						cls_token=_lowerCAmelCase	,						mask_token=_lowerCAmelCase	,						tokenize_chinese_chars=_lowerCAmelCase	,						strip_accents=_lowerCAmelCase	,						**_lowerCAmelCase	,						)

          UpperCAmelCase__      :  int				      =			json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("""lowercase"""	,						_lowerCAmelCase		) != do_lower_case
              or normalizer_state.get("""strip_accents"""	,						_lowerCAmelCase		) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""	,						_lowerCAmelCase		) != tokenize_chinese_chars
          ):
               UpperCAmelCase__      :  int				      =			getattr(_lowerCAmelCase	,						normalizer_state.pop("""type"""		)		)
               UpperCAmelCase__      :  List[str]				      =			do_lower_case
               UpperCAmelCase__      :  Any				      =			strip_accents
               UpperCAmelCase__      :  str				      =			tokenize_chinese_chars
               UpperCAmelCase__      :  List[Any]				      =			normalizer_class(**_lowerCAmelCase		)

          UpperCAmelCase__      :  Any				      =			do_lower_case




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None		):
          UpperCAmelCase__      :  Dict				      =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  str				      =			[self.sep_token_id]
          UpperCAmelCase__      :  Union[str, Any]				      =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  Tuple				      =			self._tokenizer.model.save(_lowerCAmelCase	,						name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)













import pytest


_UpperCAmelCase							:			Optional[int]        =							"""__dummy_dataset1__"""

_UpperCAmelCase							:			Dict        =							"""
import json
import os

import datasets


REPO_URL = \"https://huggingface.co/datasets/albertvillanova/tests-raw-jsonl/resolve/main/\"
URLS = {\"train\": REPO_URL + \"wikiann-bn-train.jsonl\", \"validation\": REPO_URL + \"wikiann-bn-validation.jsonl\"}


class __DummyDataset1__(datasets.GeneratorBasedBuilder):

    def _info(self):
        features = datasets.Features(
            {
                \"tokens\": datasets.Sequence(datasets.Value(\"string\")),
                \"ner_tags\": datasets.Sequence(
                    datasets.features.ClassLabel(
                        names=[
                            \"O\",
                            \"B-PER\",
                            \"I-PER\",
                            \"B-ORG\",
                            \"I-ORG\",
                            \"B-LOC\",
                            \"I-LOC\",
                        ]
                    )
                ),
                \"langs\": datasets.Sequence(datasets.Value(\"string\")),
                \"spans\": datasets.Sequence(datasets.Value(\"string\")),
            }
        )
        return datasets.DatasetInfo(features=features)

    def _split_generators(self, dl_manager):
        dl_path = dl_manager.download(URLS)
        return [
            datasets.SplitGenerator(datasets.Split.TRAIN, gen_kwargs={\"filepath\": dl_path[\"train\"]}),
            datasets.SplitGenerator(datasets.Split.VALIDATION, gen_kwargs={\"filepath\": dl_path[\"validation\"]}),
        ]

    def _generate_examples(self, filepath):
        with open(filepath, \"r\", encoding=\"utf-8\") as f:
            for i, line in enumerate(f):
                yield i, json.loads(line)
"""


@pytest.fixture
def 			SCREAMING_SNAKE_CASE (			)     ->	Dict:
    return DATASET_LOADING_SCRIPT_NAME


@pytest.fixture
def 			SCREAMING_SNAKE_CASE (			)     ->	List[Any]:
    return DATASET_LOADING_SCRIPT_CODE


@pytest.fixture
def 			SCREAMING_SNAKE_CASE (			_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase					)     ->	List[Any]:
    lowerCamelCase__       : int   					=						dataset_loading_script_name
    lowerCamelCase__       : Any   					=						tmp_path / 'datasets' / script_name
    script_dir.mkdir(parents=_UpperCAmelCase					)
    lowerCamelCase__       : str   					=						script_dir / F"""{script_name}.py"""
    with open(_UpperCAmelCase		,      'w'					) as f:
        f.write(_UpperCAmelCase					)
    return str(_UpperCAmelCase					)





