





import logging
import os
import threading
import time


try:
       import warnings
except ImportError:
       lowercase    : List[str]    	=  None

try:
       import msvcrt
except ImportError:
       lowercase    : str    	=  None

try:
       import fcntl
except ImportError:
       lowercase    : Optional[Any]    	=  None


# Backward compatibility
# ------------------------------------------------
try:
       TimeoutError
except NameError:
       lowercase    : int    	=  OSError


# Data
# ------------------------------------------------
lowercase    : str    	=  [
    """Timeout""",
    """BaseFileLock""",
    """WindowsFileLock""",
    """UnixFileLock""",
    """SoftFileLock""",
    """FileLock""",
]

lowercase    : int    	=  """3.0.12"""


lowercase    : Any    	=  None


def     SCREAMING_SNAKE_CASE			(      ):
       global _logger
       _UpperCamelCase              =						_logger or logging.getLogger(__name__					)
       return _logger

class  __A(						__UpperCAmelCase ):

     def __init__(      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						lock_file
            return None



     def __str__(      self      ):





            """simple docstring"""

            _UpperCamelCase              =						F'''The file lock \'{self.lock_file}\' could not be acquired.'''
            return temp






class  __A:

     def __init__(      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						lock
            return None

     def __enter__(      self      ):





            """simple docstring"""

            return self.lock



     def __exit__(      self,       A,       A,       A      ):





            """simple docstring"""

            self.lock.release()
            return None






class  __A:

     def __init__(      self,       A,       A=-1,       A=None      ):





            """simple docstring"""

            _UpperCamelCase              =						max_filename_length if max_filename_length is not None else 255
            # Hash the filename if it's too long
            _UpperCamelCase              =						self.hash_filename_if_too_long(A,       A      )
            # The path to the lock file.
            _UpperCamelCase              =						lock_file

            # The file descriptor for the *_lock_file* as it is returned by the
            # os.open() function.
            # This file lock is only NOT None, if the object currently holds the
            # lock.
            _UpperCamelCase              =						None

            # The default timeout value.
            _UpperCamelCase              =						timeout

            # We use this lock primarily for the lock counter.
            _UpperCamelCase              =						threading.Lock()

            # The lock counter is used for implementing the nested locking
            # mechanism. Whenever the lock is acquired, the counter is increased and
            # the lock is only released, when this value is 0 again.
            _UpperCamelCase              =						0
            return None

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return self._lock_file

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return self._timeout

     @timeout.setter
     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						float(A      )
            return None

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            raise NotImplementedError()

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            raise NotImplementedError()

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return self._lock_file_fd is not None

     def 	_UpperCamelCase     (      self,       A=None,       A=0.05      ):





            """simple docstring"""

            if timeout is None:
                   _UpperCamelCase              =						self.timeout

            # Increment the number right at the beginning.
            # We can still undo it, if something fails.
            with self._thread_lock:
                   self._lock_counter += 1

            _UpperCamelCase              =						id(self      )
            _UpperCamelCase              =						self._lock_file
            _UpperCamelCase              =						time.time()
            try:
                   while True:
                          with self._thread_lock:
                                 if not self.is_locked:
                                        logger().debug(F'''Attempting to acquire lock {lock_id} on {lock_filename}'''      )
                                        self._acquire()

                          if self.is_locked:
                                 logger().debug(F'''Lock {lock_id} acquired on {lock_filename}'''      )
                                 break
                          elif timeout >= 0 and time.time() - start_time > timeout:
                                 logger().debug(F'''Timeout on acquiring lock {lock_id} on {lock_filename}'''      )
                                 raise Timeout(self._lock_file      )
                          else:
                                 logger().debug(
                                     F'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...'''      )
                                 time.sleep(A      )
            except:  # noqa
                   # Something did go wrong, so decrement the counter.
                   with self._thread_lock:
                          _UpperCamelCase              =						max(0,       self._lock_counter - 1      )

                   raise
            return _Acquire_ReturnProxy(lock=self      )

     def 	_UpperCamelCase     (      self,       A=False      ):





            """simple docstring"""

            with self._thread_lock:
                   if self.is_locked:
                          self._lock_counter -= 1

                          if self._lock_counter == 0 or force:
                                 _UpperCamelCase              =						id(self      )
                                 _UpperCamelCase              =						self._lock_file

                                 logger().debug(F'''Attempting to release lock {lock_id} on {lock_filename}'''      )
                                 self._release()
                                 _UpperCamelCase              =						0
                                 logger().debug(F'''Lock {lock_id} released on {lock_filename}'''      )

            return None

     def __enter__(      self      ):





            """simple docstring"""

            self.acquire()
            return self

     def __exit__(      self,       A,       A,       A      ):





            """simple docstring"""

            self.release()
            return None

     def __del__(      self      ):





            """simple docstring"""

            self.release(force=A      )
            return None



     def 	_UpperCamelCase     (      self,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						os.path.basename(A      )
            if len(A      ) > max_length and max_length > 0:
                   _UpperCamelCase              =						os.path.dirname(A      )
                   _UpperCamelCase              =						str(hash(A      )      )
                   _UpperCamelCase              =						filename[: max_length - len(A      ) - 8] + '''...''' + hashed_filename + '''.lock'''
                   return os.path.join(A,       A      )
            else:
                   return path






class  __A(						__UpperCAmelCase ):

     def __init__(      self,       A,       A=-1,       A=None      ):





            """simple docstring"""

            from .file_utils import relative_to_absolute_path

            super().__init__(A,       timeout=A,       max_filename_length=A      )
            _UpperCamelCase              =						'''\\\\?\\''' + relative_to_absolute_path(self.lock_file      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						os.O_RDWR | os.O_CREAT | os.O_TRUNC

            try:
                   _UpperCamelCase              =						os.open(self._lock_file,       A      )
            except OSError:
                   pass
            else:
                   try:
                          msvcrt.locking(A,       msvcrt.LK_NBLCK,       1      )
                   except OSError:
                          os.close(A      )
                   else:
                          _UpperCamelCase              =						fd
            return None



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self._lock_file_fd
            _UpperCamelCase              =						None
            msvcrt.locking(A,       msvcrt.LK_UNLCK,       1      )
            os.close(A      )

            try:
                   os.remove(self._lock_file      )
            # Probably another instance of the application
            # that acquired the file lock.
            except OSError:
                   pass
            return None






class  __A(						__UpperCAmelCase ):

     def __init__(      self,       A,       A=-1,       A=None      ):





            """simple docstring"""

            _UpperCamelCase              =						os.statvfs(os.path.dirname(A      )      ).f_namemax
            super().__init__(A,       timeout=A,       max_filename_length=A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						os.O_RDWR | os.O_CREAT | os.O_TRUNC
            _UpperCamelCase              =						os.open(self._lock_file,       A      )

            try:
                   fcntl.flock(A,       fcntl.LOCK_EX | fcntl.LOCK_NB      )
            except OSError:
                   os.close(A      )
            else:
                   _UpperCamelCase              =						fd
            return None



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self._lock_file_fd
            _UpperCamelCase              =						None
            fcntl.flock(A,       fcntl.LOCK_UN      )
            os.close(A      )
            return None






class  __A(						__UpperCAmelCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
            try:
                   _UpperCamelCase              =						os.open(self._lock_file,       A      )
            except OSError:
                   pass
            else:
                   _UpperCamelCase              =						fd
            return None



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            os.close(self._lock_file_fd      )
            _UpperCamelCase              =						None

            try:
                   os.remove(self._lock_file      )
            # The file is already deleted and that's what we want.
            except OSError:
                   pass
            return None






lowercase    : Optional[Any]    	=  None

if msvcrt:
       lowercase    : Dict    	=  WindowsFileLock
elif fcntl:
       lowercase    : List[Any]    	=  UnixFileLock
else:
       lowercase    : List[str]    	=  SoftFileLock

       if warnings is not None:
              warnings.warn("""only soft file lock is available""")













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_big_bird import BigBirdTokenizer
else:
       lowercase    : Union[str, Any]    	=  None

lowercase    : Union[str, Any]    	=  logging.get_logger(__name__)
lowercase    : Tuple    	=  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

lowercase    : Any    	=  {
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    },
    """tokenizer_file""": {
        """google/bigbird-roberta-base""": (
            """https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase    : Tuple    	=  {
    """google/bigbird-roberta-base""": 4096,
    """google/bigbird-roberta-large""": 4096,
    """google/bigbird-base-trivia-itc""": 4096,
}


lowercase    : List[str]    	=  """▁"""

class  __A(						__UpperCAmelCase ):
     __A	  =       VOCAB_FILES_NAMES
     __A	  =       PRETRAINED_VOCAB_FILES_MAP
     __A	  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A	  =       BigBirdTokenizer
     __A	  =       ["input_ids", "attention_mask"]
     __A	  =       []

     def __init__(      self,       A=None,       A=None,       A="<unk>",       A="<s>",       A="</s>",       A="<pad>",       A="[SEP]",       A="[MASK]",       A="[CLS]",       **A,       ):





            """simple docstring"""

            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else bos_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else eos_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else unk_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else pad_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else cls_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else sep_token

            # Mask token behave like a normal word, i.e. include the space before it
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else mask_token

            super().__init__(
                A,       tokenizer_file=A,       bos_token=A,       eos_token=A,       unk_token=A,       sep_token=A,       pad_token=A,       cls_token=A,       mask_token=A,       **A,       )

            _UpperCamelCase              =						vocab_file
            _UpperCamelCase              =						False if not self.vocab_file else True

     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            _UpperCamelCase              =						[self.sep_token_id]
            _UpperCamelCase              =						[self.cls_token_id]
            if token_ids_a is None:
                   return cls + token_ids_a + sep
            return cls + token_ids_a + sep + token_ids_a + sep

     def 	_UpperCamelCase     (      self,       A,       A = None,       A = False      ):





            """simple docstring"""

            if already_has_special_tokens:
                   if token_ids_a is not None:
                          raise ValueError(
                              '''You should not supply a second sequence if the provided sequence of '''
                              '''ids is already formatted with special tokens for the model.'''      )
                   return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

            if token_ids_a is None:
                   return [1] + ([0] * len(A      )) + [1]
            return [1] + ([0] * len(A      )) + [1] + ([0] * len(A      )) + [1]

     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            _UpperCamelCase              =						[self.sep_token_id]
            _UpperCamelCase              =						[self.cls_token_id]

            if token_ids_a is None:
                   return len(cls + token_ids_a + sep      ) * [0]
            return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                       '''tokenizer.'''      )

            if not os.path.isdir(A      ):
                   logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
                   return
            _UpperCamelCase              =						os.path.join(
                A,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

            if os.path.abspath(self.vocab_file      ) != os.path.abspath(A      ):
                   copyfile(self.vocab_file,       A      )

            return (out_vocab_file,)






