import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class       __lowerCAmelCase     (				a_,		unittest.TestCase  ):



 '''simple docstring'''
 _lowerCamelCase  :		Dict 			=       ElectraTokenizer
 _lowerCamelCase  :		Any 			=       ElectraTokenizerFast
 _lowerCamelCase  :		Optional[int] 			=       True
 _lowerCamelCase  :		List[str] 			=       True
 _lowerCamelCase  :		List[Any] 			=       filter_non_english

 def     __snake_case    (	self				:						int     )       ->							Union[str, Any]:
        super().setUp()

        _UpperCamelCase   :				Any       =     [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        _UpperCamelCase   :				int       =     os.path.join(self.tmpdirname				,	VOCAB_FILES_NAMES['vocab_file']     )
        with open(self.vocab_file				,	'w'				,	encoding='utf-8'     ) as vocab_writer:
               vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )

 def     __snake_case    (	self				:						Union[str, Any]				,	lowerCamelCase				:						Tuple     )       ->							List[Any]:
        _UpperCamelCase   :				Dict       =     'UNwant\u00E9d,running'
        _UpperCamelCase   :				Union[str, Any]       =     'unwanted, running'
        return input_text, output_text

 def     __snake_case    (	self				:						int     )       ->							Optional[Any]:
        _UpperCamelCase   :				Union[str, Any]       =     self.tokenizer_class(self.vocab_file     )

        _UpperCamelCase   :				Optional[int]       =     tokenizer.tokenize('UNwant\u00E9d,running'     )
        self.assertListEqual(lowerCamelCase				,	['un', '##want', '##ed', ',', 'runn', '##ing']     )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase     )				,	[9, 6, 7, 1_2, 1_0, 1_1]     )

 def     __snake_case    (	self				:						List[str]     )       ->							Any:
        if not self.test_rust_tokenizer:
               return

        _UpperCamelCase   :				Dict       =     self.get_tokenizer()
        _UpperCamelCase   :				Any       =     self.get_rust_tokenizer()

        _UpperCamelCase   :				Union[str, Any]       =     'UNwant\u00E9d,running'

        _UpperCamelCase   :				Tuple       =     tokenizer.tokenize(lowerCamelCase     )
        _UpperCamelCase   :				Union[str, Any]       =     rust_tokenizer.tokenize(lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

        _UpperCamelCase   :				str       =     tokenizer.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
        _UpperCamelCase   :				Tuple       =     rust_tokenizer.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

        _UpperCamelCase   :				Dict       =     self.get_rust_tokenizer()
        _UpperCamelCase   :				Any       =     tokenizer.encode(lowerCamelCase     )
        _UpperCamelCase   :				Dict       =     rust_tokenizer.encode(lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

        # With lower casing
        _UpperCamelCase   :				List[Any]       =     self.get_tokenizer(do_lower_case=lowerCamelCase     )
        _UpperCamelCase   :				List[str]       =     self.get_rust_tokenizer(do_lower_case=lowerCamelCase     )

        _UpperCamelCase   :				int       =     'UNwant\u00E9d,running'

        _UpperCamelCase   :				Optional[int]       =     tokenizer.tokenize(lowerCamelCase     )
        _UpperCamelCase   :				Optional[Any]       =     rust_tokenizer.tokenize(lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

        _UpperCamelCase   :				Optional[int]       =     tokenizer.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
        _UpperCamelCase   :				Dict       =     rust_tokenizer.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

        _UpperCamelCase   :				int       =     self.get_rust_tokenizer()
        _UpperCamelCase   :				List[str]       =     tokenizer.encode(lowerCamelCase     )
        _UpperCamelCase   :				Optional[Any]       =     rust_tokenizer.encode(lowerCamelCase     )
        self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

 def     __snake_case    (	self				:						Any     )       ->							str:
        _UpperCamelCase   :				Any       =     BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'     )				,	['ah', '\u535A', '\u63A8', 'zz']     )

 def     __snake_case    (	self				:						List[Any]     )       ->							Union[str, Any]:
        _UpperCamelCase   :				Any       =     BasicTokenizer(do_lower_case=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )				,	['hello', '!', 'how', 'are', 'you', '?']     )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,	['hello']     )

 def     __snake_case    (	self				:						List[Any]     )       ->							List[Any]:
        _UpperCamelCase   :				str       =     BasicTokenizer(do_lower_case=lowerCamelCase				,	strip_accents=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,	['hällo', '!', 'how', 'are', 'you', '?']     )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,	['h\u00E9llo']     )

 def     __snake_case    (	self				:						Optional[Any]     )       ->							Union[str, Any]:
        _UpperCamelCase   :				List[Any]       =     BasicTokenizer(do_lower_case=lowerCamelCase				,	strip_accents=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,	['hallo', '!', 'how', 'are', 'you', '?']     )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,	['hello']     )

 def     __snake_case    (	self				:						str     )       ->							List[str]:
        _UpperCamelCase   :				List[str]       =     BasicTokenizer(do_lower_case=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,	['hallo', '!', 'how', 'are', 'you', '?']     )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,	['hello']     )

 def     __snake_case    (	self				:						Tuple     )       ->							List[Any]:
        _UpperCamelCase   :				List[Any]       =     BasicTokenizer(do_lower_case=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )				,	['HeLLo', '!', 'how', 'Are', 'yoU', '?']     )

 def     __snake_case    (	self				:						Any     )       ->							Tuple:
        _UpperCamelCase   :				Tuple       =     BasicTokenizer(do_lower_case=lowerCamelCase				,	strip_accents=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,	['HäLLo', '!', 'how', 'Are', 'yoU', '?']     )

 def     __snake_case    (	self				:						str     )       ->							str:
        _UpperCamelCase   :				Union[str, Any]       =     BasicTokenizer(do_lower_case=lowerCamelCase				,	strip_accents=lowerCamelCase     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,	['HaLLo', '!', 'how', 'Are', 'yoU', '?']     )

 def     __snake_case    (	self				:						Tuple     )       ->							List[str]:
        _UpperCamelCase   :				int       =     BasicTokenizer(do_lower_case=lowerCamelCase				,	never_split=['[UNK]']     )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'     )				,	['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']     )

 def     __snake_case    (	self				:						int     )       ->							str:
        _UpperCamelCase   :				List[Any]       =     ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

        _UpperCamelCase   :				str       =     {}
        for i, token in enumerate(lowerCamelCase     ):
               _UpperCamelCase   :				Optional[Any]       =     i
        _UpperCamelCase   :				Any       =     WordpieceTokenizer(vocab=lowerCamelCase				,	unk_token='[UNK]'     )

        self.assertListEqual(tokenizer.tokenize(''     )				,	[]     )

        self.assertListEqual(tokenizer.tokenize('unwanted running'     )				,	['un', '##want', '##ed', 'runn', '##ing']     )

        self.assertListEqual(tokenizer.tokenize('unwantedX running'     )				,	['[UNK]', 'runn', '##ing']     )

 def     __snake_case    (	self				:						Tuple     )       ->							List[Any]:
        self.assertTrue(_is_whitespace(' '     )     )
        self.assertTrue(_is_whitespace('\t'     )     )
        self.assertTrue(_is_whitespace('\r'     )     )
        self.assertTrue(_is_whitespace('\n'     )     )
        self.assertTrue(_is_whitespace('\u00A0'     )     )

        self.assertFalse(_is_whitespace('A'     )     )
        self.assertFalse(_is_whitespace('-'     )     )

 def     __snake_case    (	self				:						int     )       ->							Tuple:
        self.assertTrue(_is_control('\u0005'     )     )

        self.assertFalse(_is_control('A'     )     )
        self.assertFalse(_is_control(' '     )     )
        self.assertFalse(_is_control('\t'     )     )
        self.assertFalse(_is_control('\r'     )     )

 def     __snake_case    (	self				:						Tuple     )       ->							Union[str, Any]:
        self.assertTrue(_is_punctuation('-'     )     )
        self.assertTrue(_is_punctuation('$'     )     )
        self.assertTrue(_is_punctuation('`'     )     )
        self.assertTrue(_is_punctuation('.'     )     )

        self.assertFalse(_is_punctuation('A'     )     )
        self.assertFalse(_is_punctuation(' '     )     )

 def     __snake_case    (	self				:						Tuple     )       ->							Dict:
        _UpperCamelCase   :				List[str]       =     self.get_tokenizer()
        _UpperCamelCase   :				Optional[int]       =     self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(lowerCamelCase     ) for t in ['Test', '\xad', 'test']]				,	[['[UNK]'], [], ['[UNK]']]     )

        self.assertListEqual(
            [rust_tokenizer.tokenize(lowerCamelCase     ) for t in ['Test', '\xad', 'test']]				,	[['[UNK]'], [], ['[UNK]']]     )

 @slow
 def     __snake_case    (	self				:						int     )       ->							Optional[int]:
        _UpperCamelCase   :				List[str]       =     self.tokenizer_class.from_pretrained('google/electra-base-discriminator'     )

        _UpperCamelCase   :				int       =     tokenizer.encode('sequence builders'				,	add_special_tokens=lowerCamelCase     )
        _UpperCamelCase   :				Optional[int]       =     tokenizer.encode('multi-sequence build'				,	add_special_tokens=lowerCamelCase     )

        _UpperCamelCase   :				Tuple       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase     )
        _UpperCamelCase   :				str       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase				,	lowerCamelCase     )

        assert encoded_sentence == [1_0_1] + text + [1_0_2]
        assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

 def     __snake_case    (	self				:						Union[str, Any]     )       ->							int:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"     ):
                      _UpperCamelCase   :				Tuple       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )

                      _UpperCamelCase   :				Union[str, Any]       =     f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                      _UpperCamelCase   :				Any       =     tokenizer_r.encode_plus(
                          lowerCamelCase				,	return_attention_mask=lowerCamelCase				,	return_token_type_ids=lowerCamelCase				,	return_offsets_mapping=lowerCamelCase				,	add_special_tokens=lowerCamelCase				,	)

                      _UpperCamelCase   :				Union[str, Any]       =     tokenizer_r.do_lower_case if hasattr(lowerCamelCase				,	'do_lower_case'     ) else False
                      _UpperCamelCase   :				Dict       =     (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), 'A'),
                              ((1, 2), ','),
                              ((3, 5), 'na'),
                              ((5, 6), '##ï'),
                              ((6, 8), '##ve'),
                              ((9, 1_5), tokenizer_r.mask_token),
                              ((1_6, 2_1), 'Allen'),
                              ((2_1, 2_3), '##NL'),
                              ((2_3, 2_4), '##P'),
                              ((2_5, 3_3), 'sentence'),
                              ((3_3, 3_4), '.'),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), 'a'),
                              ((1, 2), ','),
                              ((3, 8), 'naive'),
                              ((9, 1_5), tokenizer_r.mask_token),
                              ((1_6, 2_1), 'allen'),
                              ((2_1, 2_3), '##nl'),
                              ((2_3, 2_4), '##p'),
                              ((2_5, 3_3), 'sentence'),
                              ((3_3, 3_4), '.'),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results]				,	tokenizer_r.convert_ids_to_tokens(tokens['input_ids']     )     )
                      self.assertEqual([e[0] for e in expected_results]				,	tokens['offset_mapping']     )

 def     __snake_case    (	self				:						Optional[int]     )       ->							List[Any]:
        _UpperCamelCase   :				Any       =     ['的', '人', '有']
        _UpperCamelCase   :				List[Any]       =     ''.join(lowerCamelCase     )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"     ):
                      _UpperCamelCase   :				Any       =     True
                      _UpperCamelCase   :				Union[str, Any]       =     self.tokenizer_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )
                      _UpperCamelCase   :				Union[str, Any]       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )

                      _UpperCamelCase   :				Tuple       =     tokenizer_p.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
                      _UpperCamelCase   :				Optional[Any]       =     tokenizer_r.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )

                      _UpperCamelCase   :				Optional[Any]       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase     )
                      _UpperCamelCase   :				Union[str, Any]       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase     )

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )
                      self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )

                      _UpperCamelCase   :				str       =     False
                      _UpperCamelCase   :				Optional[int]       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )
                      _UpperCamelCase   :				List[str]       =     self.tokenizer_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )

                      _UpperCamelCase   :				str       =     tokenizer_r.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )
                      _UpperCamelCase   :				Union[str, Any]       =     tokenizer_p.encode(lowerCamelCase				,	add_special_tokens=lowerCamelCase     )

                      _UpperCamelCase   :				Any       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase     )
                      _UpperCamelCase   :				int       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase     )

                      # it is expected that only the first Chinese character is not preceded by "##".
                      _UpperCamelCase   :				Union[str, Any]       =     [
                          f"##{token}" if idx != 0 else token for idx, token in enumerate(lowerCamelCase     )
                      ]
                      self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )
                      self.assertListEqual(lowerCamelCase				,	lowerCamelCase     )




import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
 from .testing_utils import (
     floats_tensor,
     load_hf_numpy,
     load_image,
     load_numpy,
     load_pt,
     nightly,
     parse_flag_from_env,
     print_tensor_test,
     require_torch_a,
     require_torch_gpu,
     skip_mps,
     slow,
     torch_all_close,
     torch_device,
 )
 from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


UpperCAmelCase							= get_logger(__name__)


def SCREAMING_SNAKE_CASE (   lowercase_:		Any ):
       if version.parse(lowercase_ ) < version.parse(lowercase_ ):
              if "dev" in min_version:
                     _UpperCamelCase   :				Tuple       =     (
                         'This example requires a source install from HuggingFace diffusers (see '
                         '`https://huggingface.co/docs/diffusers/installation#install-from-source`),'
                     )
              else:
                     _UpperCamelCase   :				str       =     F"This example requires a minimum version of {min_version},"
              error_message += F" but the version found is {__version__}.\n"
              raise ImportError(lowercase_ )



