





import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase    : List[str]    	=  logging.get_logger(__name__)

lowercase    : Any    	=  {"""vocab_file""": """spiece.model"""}

lowercase    : Optional[Any]    	=  {
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    }
}

lowercase    : List[str]    	=  {
    """google/bigbird-roberta-base""": 4096,
    """google/bigbird-roberta-large""": 4096,
    """google/bigbird-base-trivia-itc""": 4096,
}

class  __A(						__UpperCAmelCase ):
     __A	  =       VOCAB_FILES_NAMES
     __A	  =       PRETRAINED_VOCAB_FILES_MAP
     __A	  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A	  =       ["input_ids", "attention_mask"]
     __A	  =       []

     def __init__(      self,       A,       A="<unk>",       A="<s>",       A="</s>",       A="<pad>",       A="[SEP]",       A="[MASK]",       A="[CLS]",       A = None,       **A,       ):





            """simple docstring"""

            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else bos_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else eos_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else unk_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else pad_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else cls_token
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else sep_token

            # Mask token behave like a normal word, i.e. include the space before it
            _UpperCamelCase              =						AddedToken(A,       lstrip=A,       rstrip=A      ) if isinstance(A,       A      ) else mask_token

            _UpperCamelCase              =						{} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                bos_token=A,       eos_token=A,       unk_token=A,       pad_token=A,       sep_token=A,       mask_token=A,       cls_token=A,       sp_model_kwargs=self.sp_model_kwargs,       **A,       )

            _UpperCamelCase              =						vocab_file

            _UpperCamelCase              =						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
            self.sp_model.Load(A      )

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return self.sp_model.get_piece_size()

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						{self.convert_ids_to_tokens(A      ): i for i in range(self.vocab_size      )}
            vocab.update(self.added_tokens_encoder      )
            return vocab

     def __getstate__(      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.__dict__.copy()
            _UpperCamelCase              =						None
            return state

     def __setstate__(      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						d

            # for backward compatibility
            if not hasattr(self,       '''sp_model_kwargs'''      ):
                   _UpperCamelCase              =						{}

            _UpperCamelCase              =						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
            self.sp_model.Load(self.vocab_file      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.sp_model.encode(A,       out_type=A      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.sp_model.piece_to_id(A      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						self.sp_model.IdToPiece(A      )
            return token

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						[]
            _UpperCamelCase              =						''''''
            _UpperCamelCase              =						False
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          if not prev_is_special:
                                 out_string += " "
                          out_string += self.sp_model.decode(A      ) + token
                          _UpperCamelCase              =						True
                          _UpperCamelCase              =						[]
                   else:
                          current_sub_tokens.append(A      )
                          _UpperCamelCase              =						False
            out_string += self.sp_model.decode(A      )
            return out_string.strip()

     def 	_UpperCamelCase     (      self,       A,       A = False,       A = None,       A = True,       **A,       ):





            """simple docstring"""

            _UpperCamelCase              =						kwargs.pop('''use_source_tokenizer''',       A      )

            _UpperCamelCase              =						self.convert_ids_to_tokens(A,       skip_special_tokens=A      )

            # To avoid mixing byte-level and unicode for byte-level BPT
            # we need to build string separately for added tokens and byte-level tokens
            # cf. https://github.com/huggingface/transformers/issues/1133
            _UpperCamelCase              =						[]
            _UpperCamelCase              =						[]
            for token in filtered_tokens:
                   if skip_special_tokens and token in self.all_special_ids:
                          continue
                   if token in self.added_tokens_encoder:
                          if current_sub_text:
                                 sub_texts.append(self.convert_tokens_to_string(A      )      )
                                 _UpperCamelCase              =						[]
                          sub_texts.append(A      )
                   else:
                          current_sub_text.append(A      )
            if current_sub_text:
                   sub_texts.append(self.convert_tokens_to_string(A      )      )

            # Mimic the behavior of the Rust tokenizer:
            # No space before [MASK] and [SEP]
            if spaces_between_special_tokens:
                   _UpperCamelCase              =						re.sub(R''' (\[(MASK|SEP)\])''',       R'''\1''',       ''' '''.join(A      )      )
            else:
                   _UpperCamelCase              =						''''''.join(A      )

            _UpperCamelCase              =						(
                clean_up_tokenization_spaces
                if clean_up_tokenization_spaces is not None
                else self.clean_up_tokenization_spaces
            )
            if clean_up_tokenization_spaces:
                   _UpperCamelCase              =						self.clean_up_tokenization(A      )
                   return clean_text
            else:
                   return text

     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            if not os.path.isdir(A      ):
                   logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
                   return
            _UpperCamelCase              =						os.path.join(
                A,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

            if os.path.abspath(self.vocab_file      ) != os.path.abspath(A      ) and os.path.isfile(self.vocab_file      ):
                   copyfile(self.vocab_file,       A      )
            elif not os.path.isfile(self.vocab_file      ):
                   with open(A,       '''wb'''      ) as fi:
                          _UpperCamelCase              =						self.sp_model.serialized_model_proto()
                          fi.write(A      )

            return (out_vocab_file,)

     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            _UpperCamelCase              =						[self.cls_token_id]
            _UpperCamelCase              =						[self.sep_token_id]
            return cls + token_ids_a + sep + token_ids_a + sep

     def 	_UpperCamelCase     (      self,       A,       A = None,       A = False      ):





            """simple docstring"""

            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=A,       token_ids_a=A,       already_has_special_tokens=A      )

            if token_ids_a is None:
                   return [1] + ([0] * len(A      )) + [1]
            return [1] + ([0] * len(A      )) + [1] + ([0] * len(A      )) + [1]



     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            _UpperCamelCase              =						[self.sep_token_id]
            _UpperCamelCase              =						[self.cls_token_id]
            if token_ids_a is None:
                   return len(cls + token_ids_a + sep      ) * [0]
            return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]













import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase    : str    	=  logging.get_logger(__name__)

lowercase    : Tuple    	=  {"""vocab_file""": """vocab.json"""}

lowercase    : int    	=  {
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

lowercase    : int    	=  {"""mgp-str""": 27}

class  __A(						__UpperCAmelCase ):
     __A	  =       VOCAB_FILES_NAMES
     __A	  =       PRETRAINED_VOCAB_FILES_MAP
     __A	  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

     def __init__(      self,       A,       A="[GO]",       A="[GO]",       A="[s]",       A="[GO]",       **A      ):





            """simple docstring"""

            super().__init__(
                unk_token=A,       bos_token=A,       eos_token=A,       pad_token=A,       **A,       )

            with open(A,       encoding='''utf-8'''      ) as vocab_handle:
                   _UpperCamelCase              =						json.load(A      )
            _UpperCamelCase              =						{v: k for k, v in self.vocab.items()}

     @property
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return len(self.vocab      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return dict(self.vocab,       **self.added_tokens_encoder      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						[]
            for s in text:
                   char_tokens.extend(A      )
            return char_tokens

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.vocab.get(A,       self.vocab.get(self.unk_token      )      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            return self.decoder.get(A      )



     def 	_UpperCamelCase     (      self,       A,       A = None      ):





            """simple docstring"""

            if not os.path.isdir(A      ):
                   logger.error('''Vocabulary path ({}) should be a directory'''.format(A      )      )
                   return
            _UpperCamelCase              =						os.path.join(
                A,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

            with open(A,       '''w''',       encoding='''utf-8'''      ) as f:
                   f.write(json.dumps(self.vocab,       indent=2,       sort_keys=A,       ensure_ascii=A      ) + '''\n'''      )

            return (vocab_file,)






