





import logging
import os
import threading
import time


try:
					import warnings
except ImportError:
					a__    :		Union[str, Any]      						=     None

try:
					import msvcrt
except ImportError:
					a__    :		Optional[Any]      						=     None

try:
					import fcntl
except ImportError:
					a__    :		Union[str, Any]      						=     None


# Backward compatibility
# ------------------------------------------------
try:
					TimeoutError
except NameError:
					a__    :		List[Any]      						=     OSError


# Data
# ------------------------------------------------
a__    :		Union[str, Any]      						=     [
    """Timeout""",
    """BaseFileLock""",
    """WindowsFileLock""",
    """UnixFileLock""",
    """SoftFileLock""",
    """FileLock""",
]

a__    :		str      						=     """3.0.12"""


a__    :		Any      						=     None

def     snake_case							():




	'''simple docstring'''







	global _logger
	lowerCamelCase__										=			_logger or logging.getLogger(__name__							)
	return _logger






class  lowercase    (      UpperCAmelCase_       ):






					"""simple docstring"""



					def __init__(    self						:     str ,					a_						:     List[str]							):

						"""simple docstring"""



						lowerCamelCase__										=			lock_file
						return None





					def __str__(    self						:     int							):

						"""simple docstring"""



						lowerCamelCase__										=			F'''The file lock \'{self.lock_file}\' could not be acquired.'''
						return temp






class  lowercase    :






					"""simple docstring"""



					def __init__(    self						:     str ,					a_						:     Dict							):

						"""simple docstring"""



						lowerCamelCase__										=			lock
						return None





					def __enter__(    self						:     Optional[Any]							):

						"""simple docstring"""



						return self.lock





					def __exit__(    self						:     Optional[Any] ,					a_						:     Union[str, Any] ,					a_						:     Tuple ,					a_						:     Tuple							):

						"""simple docstring"""



						self.lock.release()
						return None






class  lowercase    :






					"""simple docstring"""



					def __init__(    self						:     Dict ,					a_						:     str ,					a_						:     List[Any]=-1 ,					a_						:     str=None							):

						"""simple docstring"""



						lowerCamelCase__										=			max_filename_length if max_filename_length is not None else 2_55
						# Hash the filename if it's too long
						lowerCamelCase__										=			self.hash_filename_if_too_long(__UpperCAmelCase ,					__UpperCAmelCase							)
						# The path to the lock file.
						lowerCamelCase__										=			lock_file

						# The file descriptor for the *_lock_file* as it is returned by the
						# os.open() function.
						# This file lock is only NOT None, if the object currently holds the
						# lock.
						lowerCamelCase__										=			None

						# The default timeout value.
						lowerCamelCase__										=			timeout

						# We use this lock primarily for the lock counter.
						lowerCamelCase__										=			threading.Lock()

						# The lock counter is used for implementing the nested locking
						# mechanism. Whenever the lock is acquired, the counter is increased and
						# the lock is only released, when this value is 0 again.
						lowerCamelCase__										=			0
						return None





					@property
					def       _UpperCamelCase       (    self						:     int							):

						"""simple docstring"""



						return self._lock_file





					@property
					def       _UpperCamelCase       (    self						:     Optional[int]							):

						"""simple docstring"""



						return self._timeout





					@timeout.setter
					def       _UpperCamelCase       (    self						:     List[Any] ,					a_						:     Optional[int]							):

						"""simple docstring"""



						lowerCamelCase__										=			float(__UpperCAmelCase							)
						return None





					def       _UpperCamelCase       (    self						:     Union[str, Any]							):

						"""simple docstring"""



						raise NotImplementedError()





					def       _UpperCamelCase       (    self						:     List[str]							):

						"""simple docstring"""



						raise NotImplementedError()





					@property
					def       _UpperCamelCase       (    self						:     Optional[int]							):

						"""simple docstring"""



						return self._lock_file_fd is not None





					def       _UpperCamelCase       (    self						:     Union[str, Any] ,					a_						:     int=None ,					a_						:     Tuple=0.0_5							):

						"""simple docstring"""



						if timeout is None:
							lowerCamelCase__										=			self.timeout

						# Increment the number right at the beginning.
						# We can still undo it, if something fails.
						with self._thread_lock:
							self._lock_counter += 1

						lowerCamelCase__										=			id(self							)
						lowerCamelCase__										=			self._lock_file
						lowerCamelCase__										=			time.time()
						try:
							while True:
								with self._thread_lock:
									if not self.is_locked:
										logger().debug(F'''Attempting to acquire lock {lock_id} on {lock_filename}'''							)
										self._acquire()

								if self.is_locked:
									logger().debug(F'''Lock {lock_id} acquired on {lock_filename}'''							)
									break
								elif timeout >= 0 and time.time() - start_time > timeout:
									logger().debug(F'''Timeout on acquiring lock {lock_id} on {lock_filename}'''							)
									raise Timeout(self._lock_file							)
								else:
									logger().debug(
									    F'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...'''							)
									time.sleep(__UpperCAmelCase							)
						except:  # noqa
							# Something did go wrong, so decrement the counter.
							with self._thread_lock:
								lowerCamelCase__										=			max(0 ,					self._lock_counter - 1							)

							raise
						return _Acquire_ReturnProxy(lock=self							)





					def       _UpperCamelCase       (    self						:     Union[str, Any] ,					a_						:     Union[str, Any]=False							):

						"""simple docstring"""



						with self._thread_lock:
							if self.is_locked:
								self._lock_counter -= 1

								if self._lock_counter == 0 or force:
									lowerCamelCase__										=			id(self							)
									lowerCamelCase__										=			self._lock_file

									logger().debug(F'''Attempting to release lock {lock_id} on {lock_filename}'''							)
									self._release()
									lowerCamelCase__										=			0
									logger().debug(F'''Lock {lock_id} released on {lock_filename}'''							)

						return None





					def __enter__(    self						:     str							):

						"""simple docstring"""



						self.acquire()
						return self





					def __exit__(    self						:     int ,					a_						:     Optional[Any] ,					a_						:     List[Any] ,					a_						:     Dict							):

						"""simple docstring"""



						self.release()
						return None





					def __del__(    self						:     Optional[Any]							):

						"""simple docstring"""



						self.release(force=__UpperCAmelCase							)
						return None





					def       _UpperCamelCase       (    self						:     List[str] ,					a_						:     Union[str, Any] ,					a_						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			os.path.basename(__UpperCAmelCase							)
						if len(__UpperCAmelCase							) > max_length and max_length > 0:
							lowerCamelCase__										=			os.path.dirname(__UpperCAmelCase							)
							lowerCamelCase__										=			str(hash(__UpperCAmelCase							)							)
							lowerCamelCase__										=			filename[: max_length - len(__UpperCAmelCase							) - 8] + '...' + hashed_filename + '.lock'
							return os.path.join(__UpperCAmelCase ,					__UpperCAmelCase							)
						else:
							return path






class  lowercase    (      UpperCAmelCase_       ):






					"""simple docstring"""



					def __init__(    self						:     Tuple ,					a_						:     Tuple ,					a_						:     str=-1 ,					a_						:     Dict=None							):

						"""simple docstring"""



						from .file_utils import relative_to_absolute_path

						super().__init__(__UpperCAmelCase ,					timeout=__UpperCAmelCase ,					max_filename_length=__UpperCAmelCase							)
						lowerCamelCase__										=			'\\\\?\\' + relative_to_absolute_path(self.lock_file							)





					def       _UpperCamelCase       (    self						:     Optional[int]							):

						"""simple docstring"""



						lowerCamelCase__										=			os.O_RDWR | os.O_CREAT | os.O_TRUNC

						try:
							lowerCamelCase__										=			os.open(self._lock_file ,					__UpperCAmelCase							)
						except OSError:
							pass
						else:
							try:
								msvcrt.locking(__UpperCAmelCase ,					msvcrt.LK_NBLCK ,					1							)
							except OSError:
								os.close(__UpperCAmelCase							)
							else:
								lowerCamelCase__										=			fd
						return None





					def       _UpperCamelCase       (    self						:     Optional[Any]							):

						"""simple docstring"""



						lowerCamelCase__										=			self._lock_file_fd
						lowerCamelCase__										=			None
						msvcrt.locking(__UpperCAmelCase ,					msvcrt.LK_UNLCK ,					1							)
						os.close(__UpperCAmelCase							)

						try:
							os.remove(self._lock_file							)
						# Probably another instance of the application
						# that acquired the file lock.
						except OSError:
							pass
						return None






class  lowercase    (      UpperCAmelCase_       ):






					"""simple docstring"""



					def __init__(    self						:     Optional[int] ,					a_						:     Tuple ,					a_						:     Optional[int]=-1 ,					a_						:     int=None							):

						"""simple docstring"""



						lowerCamelCase__										=			os.statvfs(os.path.dirname(__UpperCAmelCase							)							).f_namemax
						super().__init__(__UpperCAmelCase ,					timeout=__UpperCAmelCase ,					max_filename_length=__UpperCAmelCase							)





					def       _UpperCamelCase       (    self						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			os.O_RDWR | os.O_CREAT | os.O_TRUNC
						lowerCamelCase__										=			os.open(self._lock_file ,					__UpperCAmelCase							)

						try:
							fcntl.flock(__UpperCAmelCase ,					fcntl.LOCK_EX | fcntl.LOCK_NB							)
						except OSError:
							os.close(__UpperCAmelCase							)
						else:
							lowerCamelCase__										=			fd
						return None





					def       _UpperCamelCase       (    self						:     Dict							):

						"""simple docstring"""



						lowerCamelCase__										=			self._lock_file_fd
						lowerCamelCase__										=			None
						fcntl.flock(__UpperCAmelCase ,					fcntl.LOCK_UN							)
						os.close(__UpperCAmelCase							)
						return None






class  lowercase    (      UpperCAmelCase_       ):






					"""simple docstring"""



					def       _UpperCamelCase       (    self						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
						try:
							lowerCamelCase__										=			os.open(self._lock_file ,					__UpperCAmelCase							)
						except OSError:
							pass
						else:
							lowerCamelCase__										=			fd
						return None





					def       _UpperCamelCase       (    self						:     Union[str, Any]							):

						"""simple docstring"""



						os.close(self._lock_file_fd							)
						lowerCamelCase__										=			None

						try:
							os.remove(self._lock_file							)
						# The file is already deleted and that's what we want.
						except OSError:
							pass
						return None

a__    :		Any      						=     None

if msvcrt:
					a__    :		List[str]      						=     WindowsFileLock
elif fcntl:
					a__    :		Union[str, Any]      						=     UnixFileLock
else:
					a__    :		Tuple      						=     SoftFileLock

					if warnings is not None:
										warnings.warn("""only soft file lock is available""")







"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__A											=						logging.get_logger(__name__)

__A											=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__A											=						{
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

__A											=						{
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}






class    _lowerCAmelCase  (	a			):
       """simple docstring"""




       __magic_name__	:str						    =   VOCAB_FILES_NAMES
       __magic_name__	:List[Any]						    =   PRETRAINED_VOCAB_FILES_MAP
       __magic_name__	:Optional[Any]						    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __magic_name__	:str						    =   ["""input_ids""", """attention_mask"""]
       __magic_name__	:Any						    =   RobertaTokenizer






       def __init__(       self       ,					__UpperCAmelCase=None       ,					__UpperCAmelCase=None       ,					__UpperCAmelCase=None       ,					__UpperCAmelCase="replace"       ,					__UpperCAmelCase="<s>"       ,					__UpperCAmelCase="</s>"       ,					__UpperCAmelCase="</s>"       ,					__UpperCAmelCase="<s>"       ,					__UpperCAmelCase="<unk>"       ,					__UpperCAmelCase="<pad>"       ,					__UpperCAmelCase="<mask>"       ,					__UpperCAmelCase=False       ,					__UpperCAmelCase=True       ,					**__UpperCAmelCase       ,					):



         '''simple docstring'''

         super().__init__(
             __UpperCAmelCase       ,					__UpperCAmelCase       ,					tokenizer_file=__UpperCAmelCase       ,					errors=__UpperCAmelCase       ,					bos_token=__UpperCAmelCase       ,					eos_token=__UpperCAmelCase       ,					sep_token=__UpperCAmelCase       ,					cls_token=__UpperCAmelCase       ,					unk_token=__UpperCAmelCase       ,					pad_token=__UpperCAmelCase       ,					mask_token=__UpperCAmelCase       ,					add_prefix_space=__UpperCAmelCase       ,					trim_offsets=__UpperCAmelCase       ,					**__UpperCAmelCase       ,					)

         lowerCAmelCase__     :Dict											=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
         if pre_tok_state.get('add_prefix_space'       ,					__UpperCAmelCase ) != add_prefix_space:
           lowerCAmelCase__     :Optional[int]											=				getattr(__UpperCAmelCase       ,					pre_tok_state.pop('type' ) )
           lowerCAmelCase__     :List[Any]											=				add_prefix_space
           lowerCAmelCase__     :str											=				pre_tok_class(**__UpperCAmelCase )

         lowerCAmelCase__     :List[str]											=				add_prefix_space

         lowerCAmelCase__     :str											=				'post_processor'
         lowerCAmelCase__     :Optional[Any]											=				getattr(self.backend_tokenizer       ,					__UpperCAmelCase       ,					__UpperCAmelCase )
         if tokenizer_component_instance:
           lowerCAmelCase__     :Optional[Any]											=				json.loads(tokenizer_component_instance.__getstate__() )

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
             lowerCAmelCase__     :Any											=				tuple(state['sep'] )
           if "cls" in state:
             lowerCAmelCase__     :int											=				tuple(state['cls'] )

           lowerCAmelCase__     :List[Any]											=				False

           if state.get('add_prefix_space'       ,					__UpperCAmelCase ) != add_prefix_space:
             lowerCAmelCase__     :Union[str, Any]											=				add_prefix_space
             lowerCAmelCase__     :Any											=				True

           if state.get('trim_offsets'       ,					__UpperCAmelCase ) != trim_offsets:
             lowerCAmelCase__     :Union[str, Any]											=				trim_offsets
             lowerCAmelCase__     :Optional[int]											=				True

           if changes_to_apply:
             lowerCAmelCase__     :str											=				getattr(__UpperCAmelCase       ,					state.pop('type' ) )
             lowerCAmelCase__     :Any											=				component_class(**__UpperCAmelCase )
             setattr(self.backend_tokenizer       ,					__UpperCAmelCase       ,					__UpperCAmelCase )






       @property
       def 	snake_case		(       self ):



         '''simple docstring'''

         if self._mask_token is None:
           if self.verbose:
             logger.error('Using mask_token, but it is not set yet.' )
           return None
         return str(self._mask_token )






       @mask_token.setter
       def 	snake_case		(       self       ,					__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :List[Any]											=				AddedToken(__UpperCAmelCase       ,					lstrip=__UpperCAmelCase       ,					rstrip=__UpperCAmelCase ) if isinstance(__UpperCAmelCase       ,					__UpperCAmelCase ) else value
         lowerCAmelCase__     :List[str]											=				value






       def 	snake_case		(       self       ,					*__UpperCAmelCase       ,					**__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Optional[Any]											=				kwargs.get('is_split_into_words'       ,					__UpperCAmelCase )
         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*__UpperCAmelCase       ,					**__UpperCAmelCase )






       def 	snake_case		(       self       ,					*__UpperCAmelCase       ,					**__UpperCAmelCase ):



         '''simple docstring'''

         lowerCAmelCase__     :Any											=				kwargs.get('is_split_into_words'       ,					__UpperCAmelCase )

         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*__UpperCAmelCase       ,					**__UpperCAmelCase )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase = None ):



         '''simple docstring'''

         lowerCAmelCase__     :Union[str, Any]											=				self._tokenizer.model.save(__UpperCAmelCase       ,					name=__UpperCAmelCase )
         return tuple(__UpperCAmelCase )






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase=None ):



         '''simple docstring'''

         lowerCAmelCase__     :str											=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
           return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase = None ):



         '''simple docstring'''

         lowerCAmelCase__     :List[Any]											=				[self.sep_token_id]
         lowerCAmelCase__     :int											=				[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]
