

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__A			:					List[Any]						=			logging.get_logger(__name__)

__A			:					Union[str, Any]						=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A			:					Tuple						=			{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__A			:					str						=			{
    'junnyu/roformer_chinese_small': 1_5_3_6,
    'junnyu/roformer_chinese_base': 1_5_3_6,
    'junnyu/roformer_chinese_char_small': 5_1_2,
    'junnyu/roformer_chinese_char_base': 5_1_2,
    'junnyu/roformer_small_discriminator': 1_2_8,
    'junnyu/roformer_small_generator': 1_2_8,
}


__A			:					List[Any]						=			{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 			_SCREAMING_SNAKE_CASE (   _lowerCamelCase		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      PRETRAINED_INIT_CONFIGURATION
   lowerCamelCase__				=      RoFormerTokenizer





   def __init__(       self					:			Optional[Any]     , __lowerCamelCase					:			Any=None     , __lowerCamelCase					:			List[str]=None     , __lowerCamelCase					:			Union[str, Any]=True     , __lowerCamelCase					:			Union[str, Any]="[UNK]"     , __lowerCamelCase					:			Any="[SEP]"     , __lowerCamelCase					:			List[Any]="[PAD]"     , __lowerCamelCase					:			Tuple="[CLS]"     , __lowerCamelCase					:			List[str]="[MASK]"     , __lowerCamelCase					:			int=True     , __lowerCamelCase					:			Tuple=None     , **__lowerCamelCase					:			Tuple     , ):
       super().__init__(
           __UpperCamelCase     , tokenizer_file=__UpperCamelCase     , do_lower_case=__UpperCamelCase     , unk_token=__UpperCamelCase     , sep_token=__UpperCamelCase     , pad_token=__UpperCamelCase     , cls_token=__UpperCamelCase     , mask_token=__UpperCamelCase     , tokenize_chinese_chars=__UpperCamelCase     , strip_accents=__UpperCamelCase     , **__UpperCamelCase     , )

       SCREAMING_SNAKE_CASE										=   json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
       if (
           pre_tok_state.get("lowercase"     , __UpperCamelCase			) != do_lower_case
           or pre_tok_state.get("strip_accents"     , __UpperCamelCase			) != strip_accents
       ):
           SCREAMING_SNAKE_CASE										=   getattr(__UpperCamelCase     , pre_tok_state.pop("type"			)			)
           SCREAMING_SNAKE_CASE										=   do_lower_case
           SCREAMING_SNAKE_CASE										=   strip_accents
           SCREAMING_SNAKE_CASE										=   pre_tok_class(**__UpperCamelCase			)

       SCREAMING_SNAKE_CASE										=   do_lower_case





   def __getstate__(       self					:			str			):
       SCREAMING_SNAKE_CASE										=   self.__dict__.copy()
       SCREAMING_SNAKE_CASE										=   BertPreTokenizer()
       return state





   def __setstate__(       self					:			Dict     , __lowerCamelCase					:			Any			):
       SCREAMING_SNAKE_CASE										=   d
       SCREAMING_SNAKE_CASE										=   self.__dict__["_tokenizer"].get_vocab()
       SCREAMING_SNAKE_CASE										=   PreTokenizer.custom(JiebaPreTokenizer(__UpperCamelCase			)			)





   def      _snake_case		(       self					:			str     , __lowerCamelCase					:			Optional[int]     , __lowerCamelCase					:			Optional[int]=None			):
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

       return output





   def      _snake_case		(       self					:			Any     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]
       if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
       return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       SCREAMING_SNAKE_CASE										=   self._tokenizer.model.save(__UpperCamelCase     , name=__UpperCamelCase			)
       return tuple(__UpperCamelCase			)





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			Optional[Any]     , __lowerCamelCase					:			Optional[int]=None     , __lowerCamelCase					:			Optional[int]=None     , __lowerCamelCase					:			List[str]=False     , **__lowerCamelCase					:			Union[str, Any]     , ):
       SCREAMING_SNAKE_CASE										=   BertPreTokenizer()
       return super().save_pretrained(__UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase     , __UpperCamelCase     , **__UpperCamelCase			)




import logging
import os
import threading
import time


try:
				import warnings
except ImportError:
				__snake_case     :Any			       =None

try:
				import msvcrt
except ImportError:
				__snake_case     :Union[str, Any]			       =None

try:
				import fcntl
except ImportError:
				__snake_case     :str			       =None


# Backward compatibility
# ------------------------------------------------
try:
				TimeoutError
except NameError:
				__snake_case     :str			       =OSError


# Data
# ------------------------------------------------
__snake_case     :Any			       =[
    'Timeout',
    'BaseFileLock',
    'WindowsFileLock',
    'UnixFileLock',
    'SoftFileLock',
    'FileLock',
]

__snake_case     :str			       ='3.0.12'


__snake_case     :str			       =None






def  lowerCamelCase_	(      )       ->							List[str]:

							'''simple docstring'''






							global _logger
							A      						=    _logger or logging.getLogger(__name__				)
							return _logger



class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  Union[str, Any]       )		->     List[Any]:
											A      						=    lock_file
											return None


				def __str__(							self      :  List[Any]       )		->     int:
											A      						=    f'''The file lock \'{self.lock_file}\' could not be acquired.'''
											return temp





class        lowerCAmelCase__					:


				def __init__(							self      :  int						, __UpperCamelCase      :  Union[str, Any]       )		->     List[str]:
											A      						=    lock
											return None


				def __enter__(							self      :  Dict       )		->     Dict:
											return self.lock


				def __exit__(							self      :  int						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  Any       )		->     Optional[int]:
											self.lock.release()
											return None





class        lowerCAmelCase__					:


				def __init__(							self      :  List[Any]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  List[Any]=-1						, __UpperCamelCase      :  Optional[Any]=None       )		->     Dict:
											A      						=    max_filename_length if max_filename_length is not None else 255
											# Hash the filename if it's too long
											A      						=    self.hash_filename_if_too_long(__UpperCamelCase						, __UpperCamelCase       )
											# The path to the lock file.
											A      						=    lock_file

											# The file descriptor for the *_lock_file* as it is returned by the
											# os.open() function.
											# This file lock is only NOT None, if the object currently holds the
											# lock.
											A      						=    None

											# The default timeout value.
											A      						=    timeout

											# We use this lock primarily for the lock counter.
											A      						=    threading.Lock()

											# The lock counter is used for implementing the nested locking
											# mechanism. Whenever the lock is acquired, the counter is increased and
											# the lock is only released, when this value is 0 again.
											A      						=    0
											return None


				@property
				def __UpperCamelCase						(							self      :  str       )		->     Union[str, Any]:
											return self._lock_file


				@property
				def __UpperCamelCase						(							self      :  Tuple       )		->     Union[str, Any]:
											return self._timeout


				@timeout.setter
				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  Any       )		->     Tuple:
											A      						=    float(__UpperCamelCase       )
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     Any:
											raise NotImplementedError()


				def __UpperCamelCase						(							self      :  int       )		->     str:
											raise NotImplementedError()


				@property
				def __UpperCamelCase						(							self      :  List[Any]       )		->     Optional[Any]:
											return self._lock_file_fd is not None


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  List[str]=None						, __UpperCamelCase      :  Any=0.0_5       )		->     Any:
											# Use the default timeout, if no timeout is provided.
											if timeout is None:
																		A      						=    self.timeout

											# Increment the number right at the beginning.
											# We can still undo it, if something fails.
											with self._thread_lock:
																		self._lock_counter += 1

											A      						=    id(self       )
											A      						=    self._lock_file
											A      						=    time.time()
											try:
																		while True:
																									with self._thread_lock:
																																if not self.is_locked:
																																							logger().debug(f'''Attempting to acquire lock {lock_id} on {lock_filename}'''       )
																																							self._acquire()

																									if self.is_locked:
																																logger().debug(f'''Lock {lock_id} acquired on {lock_filename}'''       )
																																break
																									elif timeout >= 0 and time.time() - start_time > timeout:
																																logger().debug(f'''Timeout on acquiring lock {lock_id} on {lock_filename}'''       )
																																raise Timeout(self._lock_file       )
																									else:
																																logger().debug(
																																    f'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...'''       )
																																time.sleep(__UpperCamelCase       )
											except:  # noqa
																		# Something did go wrong, so decrement the counter.
																		with self._thread_lock:
																									A      						=    max(0						, self._lock_counter - 1       )

																		raise
											return _Acquire_ReturnProxy(lock=self       )


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Tuple=False       )		->     Tuple:
											with self._thread_lock:
																		if self.is_locked:
																									self._lock_counter -= 1

																									if self._lock_counter == 0 or force:
																																A      						=    id(self       )
																																A      						=    self._lock_file

																																logger().debug(f'''Attempting to release lock {lock_id} on {lock_filename}'''       )
																																self._release()
																																A      						=    0
																																logger().debug(f'''Lock {lock_id} released on {lock_filename}'''       )

											return None


				def __enter__(							self      :  int       )		->     Dict:
											self.acquire()
											return self


				def __exit__(							self      :  List[str]						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  str       )		->     Dict:
											self.release()
											return None


				def __del__(							self      :  Union[str, Any]       )		->     Optional[int]:
											self.release(force=__UpperCamelCase       )
											return None


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  int       )		->     str:
											A      						=    os.path.basename(__UpperCamelCase       )
											if len(__UpperCamelCase       ) > max_length and max_length > 0:
																		A      						=    os.path.dirname(__UpperCamelCase       )
																		A      						=    str(hash(__UpperCamelCase       )       )
																		A      						=    filename[: max_length - len(__UpperCamelCase       ) - 8] + '...' + hashed_filename + '.lock'
																		return os.path.join(__UpperCamelCase						, __UpperCamelCase       )
											else:
																		return path





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Tuple=-1						, __UpperCamelCase      :  Optional[Any]=None       )		->     Union[str, Any]:
											from .file_utils import relative_to_absolute_path

											super().__init__(__UpperCamelCase						, timeout=__UpperCamelCase						, max_filename_length=__UpperCamelCase       )
											A      						=    '\\\\?\\' + relative_to_absolute_path(self.lock_file       )


				def __UpperCamelCase						(							self      :  Any       )		->     Any:
											A      						=    os.O_RDWR | os.O_CREAT | os.O_TRUNC

											try:
																		A      						=    os.open(self._lock_file						, __UpperCamelCase       )
											except OSError:
																		pass
											else:
																		try:
																									msvcrt.locking(__UpperCamelCase						, msvcrt.LK_NBLCK						, 1       )
																		except OSError:
																									os.close(__UpperCamelCase       )
																		else:
																									A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[Any]:
											A      						=    self._lock_file_fd
											A      						=    None
											msvcrt.locking(__UpperCamelCase						, msvcrt.LK_UNLCK						, 1       )
											os.close(__UpperCamelCase       )

											try:
																		os.remove(self._lock_file       )
											# Probably another instance of the application
											# that acquired the file lock.
											except OSError:
																		pass
											return None





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  List[Any]						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  List[Any]=-1						, __UpperCamelCase      :  Dict=None       )		->     Dict:
											A      						=    os.statvfs(os.path.dirname(__UpperCamelCase       )       ).f_namemax
											super().__init__(__UpperCamelCase						, timeout=__UpperCamelCase						, max_filename_length=__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											A      						=    os.O_RDWR | os.O_CREAT | os.O_TRUNC
											A      						=    os.open(self._lock_file						, __UpperCamelCase       )

											try:
																		fcntl.flock(__UpperCamelCase						, fcntl.LOCK_EX | fcntl.LOCK_NB       )
											except OSError:
																		os.close(__UpperCamelCase       )
											else:
																		A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[int]       )		->     int:
											# Do not remove the lockfile:
											#
											#   https://github.com/benediktschmitt/py-filelock/issues/31
											#   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
											A      						=    self._lock_file_fd
											A      						=    None
											fcntl.flock(__UpperCamelCase						, fcntl.LOCK_UN       )
											os.close(__UpperCamelCase       )
											return None





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __UpperCamelCase						(							self      :  int       )		->     Optional[int]:
											A      						=    os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
											try:
																		A      						=    os.open(self._lock_file						, __UpperCamelCase       )
											except OSError:
																		pass
											else:
																		A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											os.close(self._lock_file_fd       )
											A      						=    None

											try:
																		os.remove(self._lock_file       )
											# The file is already deleted and that's what we want.
											except OSError:
																		pass
											return None





__snake_case     :List[str]			       =None

if msvcrt:
				__snake_case     :List[Any]			       =WindowsFileLock
elif fcntl:
				__snake_case     :Any			       =UnixFileLock
else:
				__snake_case     :Tuple			       =SoftFileLock

				if warnings is not None:
								warnings.warn('only soft file lock is available')