


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A				:       List[Any]      =  logging.get_logger(__name__)

A				:       List[Any]      =  {'vocab_file': 'spiece.model'}

A				:       Optional[int]      =  {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

A				:       Union[str, Any]      =  {
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}






class      UpperCamelCase(	_a ):
					snake_case_		:					Tuple    					=						VOCAB_FILES_NAMES
					snake_case_		:					Tuple    					=						PRETRAINED_VOCAB_FILES_MAP
					snake_case_		:					Optional[int]    					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					snake_case_		:					Optional[int]    					=						["""input_ids""", """attention_mask"""]
					snake_case_		:					List[int]    					=						[]


					def __init__(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		str="<unk>"  ,     SCREAMING_SNAKE_CASE						:		Union[str, Any]="<s>"  ,     SCREAMING_SNAKE_CASE						:		Tuple="</s>"  ,     SCREAMING_SNAKE_CASE						:		Optional[Any]="<pad>"  ,     SCREAMING_SNAKE_CASE						:		str="[SEP]"  ,     SCREAMING_SNAKE_CASE						:		int="[MASK]"  ,     SCREAMING_SNAKE_CASE						:		Any="[CLS]"  ,     SCREAMING_SNAKE_CASE						:		Optional[Dict[str, Any]] = None  ,     **SCREAMING_SNAKE_CASE						:		Union[str, Any]  ,     )      ->						None:




											'''simple docstring'''

											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else bos_token
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else eos_token
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else unk_token
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else pad_token
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else cls_token
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else sep_token

											# Mask token behave like a normal word, i.e. include the space before it
											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else mask_token

											__snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=SCREAMING_SNAKE_CASE  ,     eos_token=SCREAMING_SNAKE_CASE  ,     unk_token=SCREAMING_SNAKE_CASE  ,     pad_token=SCREAMING_SNAKE_CASE  ,     sep_token=SCREAMING_SNAKE_CASE  ,     mask_token=SCREAMING_SNAKE_CASE  ,     cls_token=SCREAMING_SNAKE_CASE  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **SCREAMING_SNAKE_CASE  ,     )

											__snake_case				=		vocab_file

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(SCREAMING_SNAKE_CASE    )


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]    )      ->						Union[str, Any]:




											'''simple docstring'''

											return self.sp_model.get_piece_size()


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						Any:




											'''simple docstring'''

											__snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab


					def __getstate__(    self						:		Any    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		self.__dict__.copy()
											__snake_case				=		None
											return state


					def __setstate__(    self						:		Any  ,     SCREAMING_SNAKE_CASE						:		Tuple    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		d

											# for backward compatibility
											if not hasattr(self  ,     "sp_model_kwargs"    ):
																	__snake_case				=		{}

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						List[str]:




											'''simple docstring'''

											return self.sp_model.encode(SCREAMING_SNAKE_CASE  ,     out_type=SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		Any    )      ->						int:




											'''simple docstring'''

											return self.sp_model.piece_to_id(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE    )
											return token


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Dict  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						Optional[int]:




											'''simple docstring'''

											__snake_case				=		[]
											__snake_case				=		""
											__snake_case				=		False
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							if not prev_is_special:
																													out_string += " "
																							out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE    ) + token
																							__snake_case				=		True
																							__snake_case				=		[]
																	else:
																							current_sub_tokens.append(SCREAMING_SNAKE_CASE    )
																							__snake_case				=		False
											out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE    )
											return out_string.strip()


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		bool = False  ,     SCREAMING_SNAKE_CASE						:		bool = None  ,     SCREAMING_SNAKE_CASE						:		bool = True  ,     **SCREAMING_SNAKE_CASE						:		List[str]  ,     )      ->						str:




											'''simple docstring'''

											__snake_case				=		kwargs.pop("use_source_tokenizer"  ,     SCREAMING_SNAKE_CASE    )

											__snake_case				=		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE  ,     skip_special_tokens=SCREAMING_SNAKE_CASE    )

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__snake_case				=		[]
											__snake_case				=		[]
											for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																							continue
																	if token in self.added_tokens_encoder:
																							if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )
																													__snake_case				=		[]
																							sub_texts.append(SCREAMING_SNAKE_CASE    )
																	else:
																							current_sub_text.append(SCREAMING_SNAKE_CASE    )
											if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )

											# Mimic the behavior of the Rust tokenizer:
											# No space before [MASK] and [SEP]
											if spaces_between_special_tokens:
																	__snake_case				=		re.sub(R" (\[(MASK|SEP)\])"  ,     R"\1"  ,     " ".join(SCREAMING_SNAKE_CASE    )    )
											else:
																	__snake_case				=		"".join(SCREAMING_SNAKE_CASE    )

											__snake_case				=		(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
																	__snake_case				=		self.clean_up_tokenization(SCREAMING_SNAKE_CASE    )
																	return clean_text
											else:
																	return text


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Dict  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		Optional[str] = None    )      ->						Tuple[str]:




											'''simple docstring'''

											if not os.path.isdir(SCREAMING_SNAKE_CASE    ):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
																	return
											__snake_case				=		os.path.join(
											    SCREAMING_SNAKE_CASE  ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE    ) and os.path.isfile(self.vocab_file    ):
																	copyfile(self.vocab_file  ,     SCREAMING_SNAKE_CASE    )
											elif not os.path.isfile(self.vocab_file    ):
																	with open(SCREAMING_SNAKE_CASE  ,     "wb"    ) as fi:
																							__snake_case				=		self.sp_model.serialized_model_proto()
																							fi.write(SCREAMING_SNAKE_CASE    )

											return (out_vocab_file,)


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											__snake_case				=		[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None  ,     SCREAMING_SNAKE_CASE						:		bool = False    )      ->						List[int]:




											'''simple docstring'''

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE  ,     token_ids_a=SCREAMING_SNAKE_CASE  ,     already_has_special_tokens=SCREAMING_SNAKE_CASE    )

											if token_ids_a is None:
																	return [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1]
											return [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1]






					def 				SCREAMING_SNAKE_CASE_	(    self						:		str  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]









A				:       Any      =  '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

A				:       Optional[int]      =  [{'type': 'code', 'content': INSTALL_CONTENT}]
A				:       Optional[int]      =  {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}





