





import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase	: Optional[Any]       =logging.get_logger(__name__)

__lowercase	: List[str]       ={"""vocab_file""": """spiece.model"""}

__lowercase	: Dict       ={
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    }
}

__lowercase	: List[str]       ={
    """google/bigbird-roberta-base""": 4096,
    """google/bigbird-roberta-large""": 4096,
    """google/bigbird-base-trivia-itc""": 4096,
}

class   A							(						a__    ):
      _snake_case				  =VOCAB_FILES_NAMES
      _snake_case				  =PRETRAINED_VOCAB_FILES_MAP
      _snake_case				  =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _snake_case				  =["""input_ids""", """attention_mask"""]
      _snake_case				  =[]



      def __init__(						self:    Optional[Any]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Optional[Any]="<unk>"		,	_lowerCAmelCase:    Union[str, Any]="<s>"		,	_lowerCAmelCase:    Dict="</s>"		,	_lowerCAmelCase:    Any="<pad>"		,	_lowerCAmelCase:    List[str]="[SEP]"		,	_lowerCAmelCase:    Optional[Any]="[MASK]"		,	_lowerCAmelCase:    int="[CLS]"		,	_lowerCAmelCase:    str = None		,	**_lowerCAmelCase:    Tuple		,	)    ->     None:
         '''simple docstring'''

         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else bos_token
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else eos_token
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else unk_token
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else pad_token
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else cls_token
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else sep_token

         # Mask token behave like a normal word, i.e. include the space before it
         UpperCAmelCase_           =AddedToken(__a		,	lstrip=__a		,	rstrip=__a							) if isinstance(__a		,	__a							) else mask_token

         UpperCAmelCase_           ={} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=__a		,	eos_token=__a		,	unk_token=__a		,	pad_token=__a		,	sep_token=__a		,	mask_token=__a		,	cls_token=__a		,	sp_model_kwargs=self.sp_model_kwargs		,	**__a		,	)

         UpperCAmelCase_           =vocab_file

         UpperCAmelCase_           =spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(__a							)



      @property
      def 						lowerCAmelCase__     (						self:    Dict							)    ->     List[Any]:
         '''simple docstring'''

         return self.sp_model.get_piece_size()



      def 						lowerCAmelCase__     (						self:    Any							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           ={self.convert_ids_to_tokens(__a							): i for i in range(self.vocab_size							)}
         vocab.update(self.added_tokens_encoder							)
         return vocab



      def __getstate__(						self:    Tuple							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =self.__dict__.copy()
         UpperCAmelCase_           =None
         return state



      def __setstate__(						self:    Optional[Any]		,	_lowerCAmelCase:    Optional[Any]							)    ->     List[str]:
         '''simple docstring'''

         UpperCAmelCase_           =d

         # for backward compatibility
         if not hasattr(self		,	"sp_model_kwargs"							):
            UpperCAmelCase_           ={}

         UpperCAmelCase_           =spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(self.vocab_file							)



      def 						lowerCAmelCase__     (						self:    Optional[int]		,	_lowerCAmelCase:    Optional[Any]							)    ->     List[str]:
         '''simple docstring'''

         return self.sp_model.encode(__a		,	out_type=__a							)



      def 						lowerCAmelCase__     (						self:    int		,	_lowerCAmelCase:    str							)    ->     Dict:
         '''simple docstring'''

         return self.sp_model.piece_to_id(__a							)



      def 						lowerCAmelCase__     (						self:    Optional[int]		,	_lowerCAmelCase:    Tuple							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.sp_model.IdToPiece(__a							)
         return token



      def 						lowerCAmelCase__     (						self:    Any		,	_lowerCAmelCase:    int							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =[]
         UpperCAmelCase_           =""""""
         UpperCAmelCase_           =False
         for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
               if not prev_is_special:
                  out_string += " "
               out_string += self.sp_model.decode(__a							) + token
               UpperCAmelCase_           =True
               UpperCAmelCase_           =[]
            else:
               current_sub_tokens.append(__a							)
               UpperCAmelCase_           =False
         out_string += self.sp_model.decode(__a							)
         return out_string.strip()



      def 						lowerCAmelCase__     (						self:    Dict		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Optional[Any] = False		,	_lowerCAmelCase:    Union[str, Any] = None		,	_lowerCAmelCase:    Dict = True		,	**_lowerCAmelCase:    Any		,	)    ->     str:
         '''simple docstring'''

         UpperCAmelCase_           =kwargs.pop("use_source_tokenizer"		,	__a							)

         UpperCAmelCase_           =self.convert_ids_to_tokens(__a		,	skip_special_tokens=__a							)

         # To avoid mixing byte-level and unicode for byte-level BPT
         # we need to build string separately for added tokens and byte-level tokens
         # cf. https://github.com/huggingface/transformers/issues/1133
         UpperCAmelCase_           =[]
         UpperCAmelCase_           =[]
         for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
               continue
            if token in self.added_tokens_encoder:
               if current_sub_text:
                  sub_texts.append(self.convert_tokens_to_string(__a							)							)
                  UpperCAmelCase_           =[]
               sub_texts.append(__a							)
            else:
               current_sub_text.append(__a							)
         if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(__a							)							)

         # Mimic the behavior of the Rust tokenizer:
         # No space before [MASK] and [SEP]
         if spaces_between_special_tokens:
            UpperCAmelCase_           =re.sub(r" (\[(MASK|SEP)\])"		,	r"\1"		,	" ".join(__a							)							)
         else:
            UpperCAmelCase_           ="""""".join(__a							)

         UpperCAmelCase_           =(
             clean_up_tokenization_spaces
             if clean_up_tokenization_spaces is not None
             else self.clean_up_tokenization_spaces
         )
         if clean_up_tokenization_spaces:
            UpperCAmelCase_           =self.clean_up_tokenization(__a							)
            return clean_text
         else:
            return text



      def 						lowerCAmelCase__     (						self:    Tuple		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Any = None							)    ->     Tuple[str]:
         '''simple docstring'''

         if not os.path.isdir(__a							):
            logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
            return
         UpperCAmelCase_           =os.path.join(
             __a		,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(__a							) and os.path.isfile(self.vocab_file							):
            copyfile(self.vocab_file		,	__a							)
         elif not os.path.isfile(self.vocab_file							):
            with open(__a		,	"wb"							) as fi:
               UpperCAmelCase_           =self.sp_model.serialized_model_proto()
               fi.write(__a							)

         return (out_vocab_file,)



      def 						lowerCAmelCase__     (						self:    Dict		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    List[str] = None							)    ->     List[int]:
         '''simple docstring'''

         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         UpperCAmelCase_           =[self.cls_token_id]
         UpperCAmelCase_           =[self.sep_token_id]
         return cls + token_ids_a + sep + token_ids_a + sep



      def 						lowerCAmelCase__     (						self:    Tuple		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Tuple = None		,	_lowerCAmelCase:    str = False							)    ->     List[int]:
         '''simple docstring'''

         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__a		,	token_ids_a=__a		,	already_has_special_tokens=__a							)

         if token_ids_a is None:
            return [1] + ([0] * len(__a							)) + [1]
         return [1] + ([0] * len(__a							)) + [1] + ([0] * len(__a							)) + [1]






      def 						lowerCAmelCase__     (						self:    Optional[Any]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    int = None							)    ->     List[int]:
         '''simple docstring'''

         UpperCAmelCase_           =[self.sep_token_id]
         UpperCAmelCase_           =[self.cls_token_id]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]










'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



