




"""simple docstring"""




import asyncio
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from distutils.util import strtobool
from functools import partial
from pathlib import Path
from typing import List, Union
from unittest import mock

import torch

from ..state import AcceleratorState, PartialState
from ..utils import (
    gather,
    is_bnb_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_mps_available,
    is_safetensors_available,
    is_tensorboard_available,
    is_torch_version,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)







def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase=False	):





			try:
						__lowercase			:  Optional[Any]								=       os.environ[key]
			except KeyError:
						# KEY isn't set, default to `default`.
						__lowercase			:  Tuple								=       default
			else:
						# KEY is set, convert it to True or False.
						try:
									__lowercase			:  Union[str, Any]								=       strtobool(UpperCAmelCase_	)
						except ValueError:
									# More values are supported, but let's keep the message simple.
									raise ValueError(f"""If set, {key} must be yes or no."""	)
			return _value


a_        =       parse_flag_from_env('RUN_SLOW', default=False)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skip('''Test was skipped'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(_run_slow_tests	,				'''test is slow'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(not torch.cuda.is_available()	,				'''test requires only a CPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(torch.cuda.is_available()	,				'''test requires a GPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_xpu_available()	,				'''test requires a XPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_mps_available()	,				'''test requires a `mps` backend support in `torch`'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(
			    is_transformers_available() and is_datasets_available()	,				'''test requires the Hugging Face suite'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_bnb_available()	,				'''test requires the bitsandbytes library'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_tpu_available()	,				'''test requires TPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(torch.cuda.device_count() == 1	,				'''test requires a GPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(torch.xpu.device_count() == 1	,				'''test requires a XPU'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(torch.cuda.device_count() > 1	,				'''test requires multiple GPUs'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(torch.xpu.device_count() > 1	,				'''test requires multiple XPUs'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_safetensors_available()	,				'''test requires safetensors'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_deepspeed_available()	,				'''test requires DeepSpeed'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_torch_version('''>='''	,				'''1.12.0'''	)	,				'''test requires torch version >= 1.12.0'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase=None	,				__UpperCamelCase=None	):





			if test_case is None:
						return partial(UpperCAmelCase_	,				version=UpperCAmelCase_	)
			return unittest.skipUnless(is_torch_version('''>='''	,				UpperCAmelCase_	)	,				f"""test requires torch version >= {version}"""	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_tensorboard_available()	,				'''test requires Tensorboard'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_wandb_available()	,				'''test requires wandb'''	)(UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(is_comet_ml_available()	,				'''test requires comet_ml'''	)(UpperCAmelCase_	)


a_        =       (
    any([is_wandb_available(), is_tensorboard_available()]) and not is_comet_ml_available()
)







def      __UpperCAmelCase     ( __UpperCamelCase	):
			return unittest.skipUnless(
			    _atleast_one_tracker_available	,				'''test requires at least one tracker to be available and for `comet_ml` to not be installed'''	,				)(UpperCAmelCase_	)

class   UpperCAmelCase_    (			unittest.TestCase		):
		UpperCamelCase							=True
		@classmethod
		def 	_lowerCamelCase							(						cls       )						->       Union[str, Any]:
					__lowercase			:  Optional[Any]								=       tempfile.mkdtemp()
		@classmethod
		def 	_lowerCamelCase							(						cls       )						->       Union[str, Any]:
					if os.path.exists(cls.tmpdir       ):
								shutil.rmtree(cls.tmpdir       )
		def 	_lowerCamelCase							(						self       )						->       int:
					if self.clear_on_setup:
								for path in Path(self.tmpdir       ).glob('''**/*'''       ):
											if path.is_file():
														path.unlink()
											elif path.is_dir():
														shutil.rmtree(SCREAMING_SNAKE_CASE__       )


class   UpperCAmelCase_    (			unittest.TestCase		):
		def 	_lowerCamelCase							(						self       )						->       Any:
					super().tearDown()
					# Reset the state of the AcceleratorState singleton.
					AcceleratorState._reset_state()
					PartialState._reset_state()


class   UpperCAmelCase_    (			unittest.TestCase		):
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[Any]:
					__lowercase			:  List[str]								=       mocks if isinstance(SCREAMING_SNAKE_CASE__				,						(tuple, list)       ) else [mocks]
					for m in self.mocks:
								m.start()
								self.addCleanup(m.stop       )







def      __UpperCAmelCase     ( __UpperCamelCase	):
			__lowercase			:  int								=       AcceleratorState()
			__lowercase			:  Tuple								=       tensor[None].clone().to(state.device	)
			__lowercase			:  List[str]								=       gather(UpperCAmelCase_	).cpu()
			__lowercase			:  str								=       tensor[0].cpu()
			for i in range(tensors.shape[0]	):
						if not torch.equal(tensors[i]	,				UpperCAmelCase_	):
									return False
			return True

class   UpperCAmelCase_    :
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_       )						->       List[Any]:
					__lowercase			:  Optional[Any]								=       returncode
					__lowercase			:  str								=       stdout
					__lowercase			:  Optional[int]								=       stderr







async def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase	):





			while True:
						__lowercase			:  List[Any]								=       await stream.readline()
						if line:
									callback(UpperCAmelCase_	)
						else:
									break







async def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase=None	,				__UpperCamelCase=None	,				__UpperCamelCase=None	,				__UpperCamelCase=False	,				__UpperCamelCase=False	):





			if echo:
						print('''\nRunning: '''	,				''' '''.join(UpperCAmelCase_	)	)

			__lowercase			:  List[str]								=       await asyncio.create_subprocess_exec(
			    cmd[0]	,				*cmd[1:]	,				stdin=UpperCAmelCase_	,				stdout=asyncio.subprocess.PIPE	,				stderr=asyncio.subprocess.PIPE	,				env=UpperCAmelCase_	,				)

			# note: there is a warning for a possible deadlock when using `wait` with huge amounts of data in the pipe
			# https://docs.python.org/3/library/asyncio-subprocess.html#asyncio.asyncio.subprocess.Process.wait
			#
			# If it starts hanging, will need to switch to the following code. The problem is that no data
			# will be seen until it's done and if it hangs for example there will be no debug info.
			# out, err = await p.communicate()
			# return _RunOutput(p.returncode, out, err)

			__lowercase			:  Union[str, Any]								=       []
			__lowercase			:  str								=       []

			def tee(__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase=""	):
						__lowercase			:  Tuple								=       line.decode('''utf-8'''	).rstrip()
						sink.append(UpperCAmelCase_	)
						if not quiet:
									print(UpperCAmelCase_	,				UpperCAmelCase_	,				file=UpperCAmelCase_	)

    # XXX: the timeout doesn't seem to make any difference here
			await asyncio.wait(
			    [
			        asyncio.create_task(_read_stream(p.stdout	,				lambda __UpperCamelCase	: tee(UpperCAmelCase_	,				UpperCAmelCase_	,				sys.stdout	,				label='''stdout:'''	)	)	),
			        asyncio.create_task(_read_stream(p.stderr	,				lambda __UpperCamelCase	: tee(UpperCAmelCase_	,				UpperCAmelCase_	,				sys.stderr	,				label='''stderr:'''	)	)	),
			    ]	,				timeout=UpperCAmelCase_	,				)
			return _RunOutput(await p.wait()	,				UpperCAmelCase_	,				UpperCAmelCase_	)







def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase=None	,				__UpperCamelCase=None	,				__UpperCamelCase=1_80	,				__UpperCamelCase=False	,				__UpperCamelCase=True	):
			__lowercase			:  List[Any]								=       asyncio.get_event_loop()
			__lowercase			:  Dict								=       loop.run_until_complete(
			    _stream_subprocess(UpperCAmelCase_	,				env=UpperCAmelCase_	,				stdin=UpperCAmelCase_	,				timeout=UpperCAmelCase_	,				quiet=UpperCAmelCase_	,				echo=UpperCAmelCase_	)	)

			__lowercase			:  Dict								=       ''' '''.join(UpperCAmelCase_	)
			if result.returncode > 0:
						__lowercase			:  str								=       '''\n'''.join(result.stderr	)
						raise RuntimeError(
						    f"""'{cmd_str}' failed with returncode {result.returncode}\n\n"""
						    f"""The combined stderr from workers follows:\n{stderr}"""	)

			return result

class   UpperCAmelCase_    (			_lowerCAmelCase		):
		pass







def      __UpperCAmelCase     ( __UpperCamelCase	,				__UpperCamelCase=False	):





			try:
						__lowercase			:  int								=       subprocess.check_output(UpperCAmelCase_	,				stderr=subprocess.STDOUT	)
						if return_stdout:
									if hasattr(UpperCAmelCase_	,				'''decode'''	):
												__lowercase			:  Optional[int]								=       output.decode('''utf-8'''	)
									return output
			except subprocess.CalledProcessError as e:
						raise SubprocessCallException(
						    f"""Command `{" ".join(UpperCAmelCase_	)}` failed with the following error:\n\n{e.output.decode()}"""	) from e








"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


UpperCamelCase     =							{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase     =							{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase     =							{
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

UpperCamelCase     =							{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}






class       UpperCamelCase__			(   _lowerCAmelCase      ):

			"""simple docstring"""


			A__	:					Dict							  =     VOCAB_FILES_NAMES
			A__	:					str							  =     PRETRAINED_VOCAB_FILES_MAP
			A__	:					List[Any]							  =     PRETRAINED_INIT_CONFIGURATION
			A__	:					Any							  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__	:					List[str]							  =     ElectraTokenizer


			def __init__(      self						,	SCREAMING_SNAKE_CASE__=None						,	SCREAMING_SNAKE_CASE__=None						,	SCREAMING_SNAKE_CASE__=True						,	SCREAMING_SNAKE_CASE__="[UNK]"						,	SCREAMING_SNAKE_CASE__="[SEP]"						,	SCREAMING_SNAKE_CASE__="[PAD]"						,	SCREAMING_SNAKE_CASE__="[CLS]"						,	SCREAMING_SNAKE_CASE__="[MASK]"						,	SCREAMING_SNAKE_CASE__=True						,	SCREAMING_SNAKE_CASE__=None						,	**SCREAMING_SNAKE_CASE__						,	)      ->	str:
				super().__init__(
				    SCREAMING_SNAKE_CASE__						,	tokenizer_file=SCREAMING_SNAKE_CASE__						,	do_lower_case=SCREAMING_SNAKE_CASE__						,	unk_token=SCREAMING_SNAKE_CASE__						,	sep_token=SCREAMING_SNAKE_CASE__						,	pad_token=SCREAMING_SNAKE_CASE__						,	cls_token=SCREAMING_SNAKE_CASE__						,	mask_token=SCREAMING_SNAKE_CASE__						,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE__						,	strip_accents=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__						,	)

				A__											=					json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
				if (
				    normalizer_state.get("lowercase"						,	SCREAMING_SNAKE_CASE__		) != do_lower_case
				    or normalizer_state.get("strip_accents"						,	SCREAMING_SNAKE_CASE__		) != strip_accents
				    or normalizer_state.get("handle_chinese_chars"						,	SCREAMING_SNAKE_CASE__		) != tokenize_chinese_chars
				):
					A__											=					getattr(SCREAMING_SNAKE_CASE__						,	normalizer_state.pop("type"		)		)
					A__											=					do_lower_case
					A__											=					strip_accents
					A__											=					tokenize_chinese_chars
					A__											=					normalizer_class(**SCREAMING_SNAKE_CASE__		)

				A__											=					do_lower_case


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__=None		)      ->	List[str]:
				A__											=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

				return output


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None		)      ->	List[int]:
				A__											=					[self.sep_token_id]
				A__											=					[self.cls_token_id]
				if token_ids_a is None:
					return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None		)      ->	Tuple[str]:
				A__											=					self._tokenizer.model.save(SCREAMING_SNAKE_CASE__						,	name=SCREAMING_SNAKE_CASE__		)
				return tuple(SCREAMING_SNAKE_CASE__		)




