


import logging
import os
import threading
import time


try:
	import warnings
except ImportError:
	A				:       int      =  None

try:
	import msvcrt
except ImportError:
	A				:       Tuple      =  None

try:
	import fcntl
except ImportError:
	A				:       Optional[int]      =  None


# Backward compatibility
# ------------------------------------------------
try:
	TimeoutError
except NameError:
	A				:       Union[str, Any]      =  OSError


# Data
# ------------------------------------------------
A				:       Tuple      =  [
    'Timeout',
    'BaseFileLock',
    'WindowsFileLock',
    'UnixFileLock',
    'SoftFileLock',
    'FileLock',
]

A				:       Any      =  '3.0.12'


A				:       str      =  None





def 						_lowerCAmelCase							(     )    ->						List[Any]:


						'''simple docstring'''






						global _logger
						__snake_case				=		_logger or logging.getLogger(__name__			)
						return _logger






class      UpperCamelCase(	UpperCAmelCase__ ):


					def __init__(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						str:




											'''simple docstring'''

											__snake_case				=		lock_file
											return None






					def __str__(    self						:		str    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		f'''The file lock \'{self.lock_file}\' could not be acquired.'''
											return temp






class      UpperCamelCase:


					def __init__(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		Any    )      ->						Union[str, Any]:




											'''simple docstring'''

											__snake_case				=		lock
											return None


					def __enter__(    self						:		List[str]    )      ->						Optional[Any]:




											'''simple docstring'''

											return self.lock






					def __exit__(    self						:		Optional[Any]  ,     SCREAMING_SNAKE_CASE						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						Tuple:




											'''simple docstring'''

											self.lock.release()
											return None






class      UpperCamelCase:


					def __init__(    self						:		str  ,     SCREAMING_SNAKE_CASE						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		str=-1  ,     SCREAMING_SNAKE_CASE						:		Optional[int]=None    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		max_filename_length if max_filename_length is not None else 2_5_5
											# Hash the filename if it's too long
											__snake_case				=		self.hash_filename_if_too_long(__lowerCAmelCase  ,     __lowerCAmelCase    )
											# The path to the lock file.
											__snake_case				=		lock_file

											# The file descriptor for the *_lock_file* as it is returned by the
											# os.open() function.
											# This file lock is only NOT None, if the object currently holds the
											# lock.
											__snake_case				=		None

											# The default timeout value.
											__snake_case				=		timeout

											# We use this lock primarily for the lock counter.
											__snake_case				=		threading.Lock()

											# The lock counter is used for implementing the nested locking
											# mechanism. Whenever the lock is acquired, the counter is increased and
											# the lock is only released, when this value is 0 again.
											__snake_case				=		0
											return None


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]    )      ->						Dict:




											'''simple docstring'''

											return self._lock_file


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]    )      ->						Optional[Any]:




											'''simple docstring'''

											return self._timeout


					@timeout.setter
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		Any    )      ->						Optional[int]:




											'''simple docstring'''

											__snake_case				=		float(__lowerCAmelCase    )
											return None


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						Dict:




											'''simple docstring'''

											raise NotImplementedError()


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]    )      ->						List[Any]:




											'''simple docstring'''

											raise NotImplementedError()


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[Any]    )      ->						Any:




											'''simple docstring'''

											return self._lock_file_fd is not None


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		List[Any]=None  ,     SCREAMING_SNAKE_CASE						:		str=0.05    )      ->						str:




											'''simple docstring'''

											if timeout is None:
																	__snake_case				=		self.timeout

											# Increment the number right at the beginning.
											# We can still undo it, if something fails.
											with self._thread_lock:
																	self._lock_counter += 1

											__snake_case				=		id(self    )
											__snake_case				=		self._lock_file
											__snake_case				=		time.time()
											try:
																	while True:
																							with self._thread_lock:
																													if not self.is_locked:
																																			logger().debug(f'''Attempting to acquire lock {lock_id} on {lock_filename}'''    )
																																			self._acquire()

																							if self.is_locked:
																													logger().debug(f'''Lock {lock_id} acquired on {lock_filename}'''    )
																													break
																							elif timeout >= 0 and time.time() - start_time > timeout:
																													logger().debug(f'''Timeout on acquiring lock {lock_id} on {lock_filename}'''    )
																													raise Timeout(self._lock_file    )
																							else:
																													logger().debug(
																													    f'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...'''    )
																													time.sleep(__lowerCAmelCase    )
											except:  # noqa
																	# Something did go wrong, so decrement the counter.
																	with self._thread_lock:
																							__snake_case				=		max(0  ,     self._lock_counter - 1    )

																	raise
											return _Acquire_ReturnProxy(lock=self    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		Tuple=False    )      ->						List[Any]:




											'''simple docstring'''

											with self._thread_lock:
																	if self.is_locked:
																							self._lock_counter -= 1

																							if self._lock_counter == 0 or force:
																													__snake_case				=		id(self    )
																													__snake_case				=		self._lock_file

																													logger().debug(f'''Attempting to release lock {lock_id} on {lock_filename}'''    )
																													self._release()
																													__snake_case				=		0
																													logger().debug(f'''Lock {lock_id} released on {lock_filename}'''    )

											return None


					def __enter__(    self						:		str    )      ->						List[str]:




											'''simple docstring'''

											self.acquire()
											return self


					def __exit__(    self						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		Tuple    )      ->						Union[str, Any]:




											'''simple docstring'''

											self.release()
											return None


					def __del__(    self						:		List[Any]    )      ->						List[Any]:




											'''simple docstring'''

											self.release(force=__lowerCAmelCase    )
											return None






					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		int    )      ->						str:




											'''simple docstring'''

											__snake_case				=		os.path.basename(__lowerCAmelCase    )
											if len(__lowerCAmelCase    ) > max_length and max_length > 0:
																	__snake_case				=		os.path.dirname(__lowerCAmelCase    )
																	__snake_case				=		str(hash(__lowerCAmelCase    )    )
																	__snake_case				=		filename[: max_length - len(__lowerCAmelCase    ) - 8] + "..." + hashed_filename + ".lock"
																	return os.path.join(__lowerCAmelCase  ,     __lowerCAmelCase    )
											else:
																	return path






class      UpperCamelCase(	UpperCAmelCase__ ):


					def __init__(    self						:		Dict  ,     SCREAMING_SNAKE_CASE						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[str]=-1  ,     SCREAMING_SNAKE_CASE						:		int=None    )      ->						int:




											'''simple docstring'''

											from .file_utils import relative_to_absolute_path

											super().__init__(__lowerCAmelCase  ,     timeout=__lowerCAmelCase  ,     max_filename_length=__lowerCAmelCase    )
											__snake_case				=		"\\\\?\\" + relative_to_absolute_path(self.lock_file    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		os.O_RDWR | os.O_CREAT | os.O_TRUNC

											try:
																	__snake_case				=		os.open(self._lock_file  ,     __lowerCAmelCase    )
											except OSError:
																	pass
											else:
																	try:
																							msvcrt.locking(__lowerCAmelCase  ,     msvcrt.LK_NBLCK  ,     1    )
																	except OSError:
																							os.close(__lowerCAmelCase    )
																	else:
																							__snake_case				=		fd
											return None






					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						Union[str, Any]:




											'''simple docstring'''

											__snake_case				=		self._lock_file_fd
											__snake_case				=		None
											msvcrt.locking(__lowerCAmelCase  ,     msvcrt.LK_UNLCK  ,     1    )
											os.close(__lowerCAmelCase    )

											try:
																	os.remove(self._lock_file    )
											# Probably another instance of the application
											# that acquired the file lock.
											except OSError:
																	pass
											return None






class      UpperCamelCase(	UpperCAmelCase__ ):


					def __init__(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		int=-1  ,     SCREAMING_SNAKE_CASE						:		Optional[int]=None    )      ->						str:




											'''simple docstring'''

											__snake_case				=		os.statvfs(os.path.dirname(__lowerCAmelCase    )    ).f_namemax
											super().__init__(__lowerCAmelCase  ,     timeout=__lowerCAmelCase  ,     max_filename_length=__lowerCAmelCase    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]    )      ->						Optional[int]:




											'''simple docstring'''

											__snake_case				=		os.O_RDWR | os.O_CREAT | os.O_TRUNC
											__snake_case				=		os.open(self._lock_file  ,     __lowerCAmelCase    )

											try:
																	fcntl.flock(__lowerCAmelCase  ,     fcntl.LOCK_EX | fcntl.LOCK_NB    )
											except OSError:
																	os.close(__lowerCAmelCase    )
											else:
																	__snake_case				=		fd
											return None






					def 				SCREAMING_SNAKE_CASE_	(    self						:		int    )      ->						Any:




											'''simple docstring'''

											__snake_case				=		self._lock_file_fd
											__snake_case				=		None
											fcntl.flock(__lowerCAmelCase  ,     fcntl.LOCK_UN    )
											os.close(__lowerCAmelCase    )
											return None






class      UpperCamelCase(	UpperCAmelCase__ ):


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]    )      ->						str:




											'''simple docstring'''

											__snake_case				=		os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
											try:
																	__snake_case				=		os.open(self._lock_file  ,     __lowerCAmelCase    )
											except OSError:
																	pass
											else:
																	__snake_case				=		fd
											return None






					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[Any]    )      ->						Dict:




											'''simple docstring'''

											os.close(self._lock_file_fd    )
											__snake_case				=		None

											try:
																	os.remove(self._lock_file    )
											# The file is already deleted and that's what we want.
											except OSError:
																	pass
											return None




A				:       Any      =  None

if msvcrt:
	A				:       Tuple      =  WindowsFileLock
elif fcntl:
	A				:       Tuple      =  UnixFileLock
else:
	A				:       Optional[int]      =  SoftFileLock

	if warnings is not None:
		warnings.warn('only soft file lock is available')











'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowercase  =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase  =			{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase  =			{
    """google/electra-small-generator""": 5_12,
    """google/electra-base-generator""": 5_12,
    """google/electra-large-generator""": 5_12,
    """google/electra-small-discriminator""": 5_12,
    """google/electra-base-discriminator""": 5_12,
    """google/electra-large-discriminator""": 5_12,
}

_lowercase  =			{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}
class 						a_				(			UpperCAmelCase__						):
	lowercase_       :     Tuple			  =      VOCAB_FILES_NAMES
	lowercase_       :     Optional[int]			  =      PRETRAINED_VOCAB_FILES_MAP
	lowercase_       :     int			  =      PRETRAINED_INIT_CONFIGURATION
	lowercase_       :     str			  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_       :     List[Any]			  =      ElectraTokenizer




	def __init__(       self     :  Optional[Any]       ,		__lowerCAmelCase     :  Optional[Any]=None       ,		__lowerCAmelCase     :  str=None       ,		__lowerCAmelCase     :  Union[str, Any]=True       ,		__lowerCAmelCase     :  int="[UNK]"       ,		__lowerCAmelCase     :  str="[SEP]"       ,		__lowerCAmelCase     :  Optional[Any]="[PAD]"       ,		__lowerCAmelCase     :  Optional[Any]="[CLS]"       ,		__lowerCAmelCase     :  str="[MASK]"       ,		__lowerCAmelCase     :  List[Any]=True       ,		__lowerCAmelCase     :  Optional[int]=None       ,		**__lowerCAmelCase     :  List[Any]       ,		):
							super().__init__(
							    __lowerCAmelCase       ,		tokenizer_file=__lowerCAmelCase       ,		do_lower_case=__lowerCAmelCase       ,		unk_token=__lowerCAmelCase       ,		sep_token=__lowerCAmelCase       ,		pad_token=__lowerCAmelCase       ,		cls_token=__lowerCAmelCase       ,		mask_token=__lowerCAmelCase       ,		tokenize_chinese_chars=__lowerCAmelCase       ,		strip_accents=__lowerCAmelCase       ,		**__lowerCAmelCase       ,		)

							__snake_case						 =   json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
							if (
							    normalizer_state.get('lowercase'       ,		__lowerCAmelCase      ) != do_lower_case
							    or normalizer_state.get('strip_accents'       ,		__lowerCAmelCase      ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'       ,		__lowerCAmelCase      ) != tokenize_chinese_chars
							):
													__snake_case						 =   getattr(__lowerCAmelCase       ,		normalizer_state.pop('type'      )      )
													__snake_case						 =   do_lower_case
													__snake_case						 =   strip_accents
													__snake_case						 =   tokenize_chinese_chars
													__snake_case						 =   normalizer_class(**__lowerCAmelCase      )

							__snake_case						 =   do_lower_case




	def   lowercase__   (       self     :  Union[str, Any]       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  Tuple=None      ):
							__snake_case						 =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output




	def   lowercase__   (       self     :  Any       ,		__lowerCAmelCase     :  List[int]       ,		__lowerCAmelCase     :  Optional[List[int]] = None      ):
							__snake_case						 =   [self.sep_token_id]
							__snake_case						 =   [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep      ) * [0]
							return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




	def   lowercase__   (       self     :  List[Any]       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  Optional[str] = None      ):
							__snake_case						 =   self._tokenizer.model.save(__lowerCAmelCase       ,		name=__lowerCAmelCase      )
							return tuple(__lowerCAmelCase      )

