



import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__       =	logging.get_logger(__name__)

UpperCAmelCase__       =	{"vocab_file": "spiece.model"}

UpperCAmelCase__       =	{
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    }
}

UpperCAmelCase__       =	{
    "google/bigbird-roberta-base": 4096,
    "google/bigbird-roberta-large": 4096,
    "google/bigbird-base-trivia-itc": 4096,
}







class       __lowerCAmelCase   (      A						):
			UpperCamelCase        =					VOCAB_FILES_NAMES
			UpperCamelCase        =					PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase        =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase        =					['''input_ids''', '''attention_mask''']
			UpperCamelCase        =					[]



			def __init__(						self       :  Dict			,		A       :  Tuple			,		A       :  Dict="<unk>"			,		A       :  Optional[int]="<s>"			,		A       :  Any="</s>"			,		A       :  List[str]="<pad>"			,		A       :  Union[str, Any]="[SEP]"			,		A       :  List[Any]="[MASK]"			,		A       :  List[str]="[CLS]"			,		A       :  Optional[Dict[str, Any]] = None			,		**A       :  int			,		)       ->   None:



						"""simple docstring"""






						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else bos_token
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else eos_token
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else unk_token
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else pad_token
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else cls_token
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else sep_token

						# Mask token behave like a normal word, i.e. include the space before it
						_UpperCAmelCase			=       AddedToken(A			,		lstrip=A			,		rstrip=A) if isinstance(A			,		A) else mask_token

						_UpperCAmelCase			=       {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=A			,		eos_token=A			,		unk_token=A			,		pad_token=A			,		sep_token=A			,		mask_token=A			,		cls_token=A			,		sp_model_kwargs=self.sp_model_kwargs			,		**A			,		)

						_UpperCAmelCase			=       vocab_file

						_UpperCAmelCase			=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(A)



			@property
			def    _lowerCamelCase	(						self       :  List[Any])       ->   Union[str, Any]:



						"""simple docstring"""






						return self.sp_model.get_piece_size()



			def    _lowerCamelCase	(						self       :  Optional[Any])       ->   Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       {self.convert_ids_to_tokens(A): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab



			def __getstate__(						self       :  Tuple)       ->   Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.__dict__.copy()
						_UpperCAmelCase			=       None
						return state



			def __setstate__(						self       :  Dict			,		A       :  Optional[Any])       ->   Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       d

						# for backward compatibility
						if not hasattr(self			,		'sp_model_kwargs'):
									_UpperCAmelCase			=       {}

						_UpperCAmelCase			=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.vocab_file)



			def    _lowerCamelCase	(						self       :  str			,		A       :  str)       ->   List[str]:



						"""simple docstring"""






						return self.sp_model.encode(A			,		out_type=A)



			def    _lowerCamelCase	(						self       :  List[str]			,		A       :  List[str])       ->   Union[str, Any]:



						"""simple docstring"""






						return self.sp_model.piece_to_id(A)



			def    _lowerCamelCase	(						self       :  List[str]			,		A       :  List[Any])       ->   List[str]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.sp_model.IdToPiece(A)
						return token



			def    _lowerCamelCase	(						self       :  List[str]			,		A       :  Union[str, Any])       ->   Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       []
						_UpperCAmelCase			=       ''
						_UpperCAmelCase			=       False
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												if not prev_is_special:
															out_string += " "
												out_string += self.sp_model.decode(A) + token
												_UpperCAmelCase			=       True
												_UpperCAmelCase			=       []
									else:
												current_sub_tokens.append(A)
												_UpperCAmelCase			=       False
						out_string += self.sp_model.decode(A)
						return out_string.strip()



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  List[int]			,		A       :  bool = False			,		A       :  bool = None			,		A       :  bool = True			,		**A       :  Tuple			,		)       ->   str:



						"""simple docstring"""






						_UpperCAmelCase			=       kwargs.pop('use_source_tokenizer'			,		A)

						_UpperCAmelCase			=       self.convert_ids_to_tokens(A			,		skip_special_tokens=A)

						# To avoid mixing byte-level and unicode for byte-level BPT
						# we need to build string separately for added tokens and byte-level tokens
						# cf. https://github.com/huggingface/transformers/issues/1133
						_UpperCAmelCase			=       []
						_UpperCAmelCase			=       []
						for token in filtered_tokens:
									if skip_special_tokens and token in self.all_special_ids:
												continue
									if token in self.added_tokens_encoder:
												if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(A))
															_UpperCAmelCase			=       []
												sub_texts.append(A)
									else:
												current_sub_text.append(A)
						if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(A))

						# Mimic the behavior of the Rust tokenizer:
						# No space before [MASK] and [SEP]
						if spaces_between_special_tokens:
									_UpperCAmelCase			=       re.sub(R' (\[(MASK|SEP)\])'			,		R'\1'			,		' '.join(A))
						else:
									_UpperCAmelCase			=       ''.join(A)

						_UpperCAmelCase			=       (
						    clean_up_tokenization_spaces
						    if clean_up_tokenization_spaces is not None
						    else self.clean_up_tokenization_spaces
						)
						if clean_up_tokenization_spaces:
									_UpperCAmelCase			=       self.clean_up_tokenization(A)
									return clean_text
						else:
									return text



			def    _lowerCamelCase	(						self       :  Optional[Any]			,		A       :  str			,		A       :  Optional[str] = None)       ->   Tuple[str]:



						"""simple docstring"""






						if not os.path.isdir(A):
									logger.error(F"Vocabulary path ({save_directory}) should be a directory")
									return
						_UpperCAmelCase			=       os.path.join(
						    A			,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

						if os.path.abspath(self.vocab_file) != os.path.abspath(A) and os.path.isfile(self.vocab_file):
									copyfile(self.vocab_file			,		A)
						elif not os.path.isfile(self.vocab_file):
									with open(A			,		'wb') as fi:
												_UpperCAmelCase			=       self.sp_model.serialized_model_proto()
												fi.write(A)

						return (out_vocab_file,)



			def    _lowerCamelCase	(						self       :  Tuple			,		A       :  List[int]			,		A       :  Optional[List[int]] = None)       ->   List[int]:



						"""simple docstring"""






						if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						_UpperCAmelCase			=       [self.cls_token_id]
						_UpperCAmelCase			=       [self.sep_token_id]
						return cls + token_ids_a + sep + token_ids_a + sep



			def    _lowerCamelCase	(						self       :  Dict			,		A       :  List[int]			,		A       :  Optional[List[int]] = None			,		A       :  bool = False)       ->   List[int]:



						"""simple docstring"""






						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=A			,		token_ids_a=A			,		already_has_special_tokens=A)

						if token_ids_a is None:
									return [1] + ([0] * len(A)) + [1]
						return [1] + ([0] * len(A)) + [1] + ([0] * len(A)) + [1]



			def    _lowerCamelCase	(						self       :  Optional[int]			,		A       :  List[int]			,		A       :  Optional[List[int]] = None)       ->   List[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       [self.sep_token_id]
						_UpperCAmelCase			=       [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCAmelCase__       =	{
    "configuration_mega": ["MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegaConfig", "MegaOnnxConfig"],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__       =	[
							    "MEGA_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "MegaForCausalLM",
							    "MegaForMaskedLM",
							    "MegaForMultipleChoice",
							    "MegaForQuestionAnswering",
							    "MegaForSequenceClassification",
							    "MegaForTokenClassification",
							    "MegaModel",
							    "MegaPreTrainedModel",
							]

if TYPE_CHECKING:
							from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_mega import (
														    MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
														    MegaForCausalLM,
														    MegaForMaskedLM,
														    MegaForMultipleChoice,
														    MegaForQuestionAnswering,
														    MegaForSequenceClassification,
														    MegaForTokenClassification,
														    MegaModel,
														    MegaPreTrainedModel,
														)

else:
							import sys

							UpperCAmelCase__       =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
