
'''simple docstring'''







import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowercase			:  int   							=	logging.get_logger(__name__)

_lowercase			:  List[Any]   							=	{"vocab_file": "spiece.model"}

_lowercase			:  List[Any]   							=	{
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    }
}

_lowercase			:  Dict   							=	{
    "google/bigbird-roberta-base": 4_0_9_6,
    "google/bigbird-roberta-large": 4_0_9_6,
    "google/bigbird-base-trivia-itc": 4_0_9_6,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):






					"""simple docstring"""




					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']
					lowerCAmelCase_					       =					[]






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE="[SEP]"    ,							__SCREAMING_SNAKE_CASE="[MASK]"    ,							__SCREAMING_SNAKE_CASE="[CLS]"    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else bos_token
											lowercase_     :					Optional[Any]													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else eos_token
											lowercase_     :					Tuple													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else unk_token
											lowercase_     :					str													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else pad_token
											lowercase_     :					Union[str, Any]													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else cls_token
											lowercase_     :					Union[str, Any]													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else sep_token

											# Mask token behave like a normal word, i.e. include the space before it
											lowercase_     :					Optional[int]													=	AddedToken(__SCREAMING_SNAKE_CASE    ,							lstrip=__SCREAMING_SNAKE_CASE    ,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) else mask_token

											lowercase_     :					List[str]													=	{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							mask_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Optional[int]													=	vocab_file

											lowercase_     :					Optional[int]													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(__SCREAMING_SNAKE_CASE		)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return self.sp_model.get_piece_size()






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	self.__dict__.copy()
											lowercase_     :					Union[str, Any]													=	None
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Dict													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					List[str]													=	{}

											lowercase_     :					List[str]													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(self.vocab_file		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.piece_to_id(__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE		)
											return token






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	[]
											lowercase_     :					int													=	''''''
											lowercase_     :					List[str]													=	False
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							if not prev_is_special:
																													out_string += " "
																							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		) + token
																							lowercase_     :					Tuple													=	True
																							lowercase_     :					Union[str, Any]													=	[]
																	else:
																							current_sub_tokens.append(__SCREAMING_SNAKE_CASE		)
																							lowercase_     :					Tuple													=	False
											out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		)
											return out_string.strip()






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = False    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = True    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Any													=	kwargs.pop('''use_source_tokenizer'''    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[Any]													=	self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE    ,							skip_special_tokens=__SCREAMING_SNAKE_CASE		)

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											lowercase_     :					Dict													=	[]
											lowercase_     :					List[str]													=	[]
											for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																							continue
																	if token in self.added_tokens_encoder:
																							if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(__SCREAMING_SNAKE_CASE		)		)
																													lowercase_     :					Union[str, Any]													=	[]
																							sub_texts.append(__SCREAMING_SNAKE_CASE		)
																	else:
																							current_sub_text.append(__SCREAMING_SNAKE_CASE		)
											if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(__SCREAMING_SNAKE_CASE		)		)

											# Mimic the behavior of the Rust tokenizer:
											# No space before [MASK] and [SEP]
											if spaces_between_special_tokens:
																	lowercase_     :					Tuple													=	re.sub(R''' (\[(MASK|SEP)\])'''    ,							R'''\1'''    ,							''' '''.join(__SCREAMING_SNAKE_CASE		)		)
											else:
																	lowercase_     :					Union[str, Any]													=	''''''.join(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					str													=	(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
																	lowercase_     :					Tuple													=	self.clean_up_tokenization(__SCREAMING_SNAKE_CASE		)
																	return clean_text
											else:
																	return text






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					Optional[int]													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.vocab_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					Optional[int]													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											lowercase_     :					Optional[int]													=	[self.cls_token_id]
											lowercase_     :					Optional[int]													=	[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE    ,							token_ids_a=__SCREAMING_SNAKE_CASE    ,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											if token_ids_a is None:
																	return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1]
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	[self.sep_token_id]
											lowercase_     :					Optional[Any]													=	[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





'''simple docstring'''







import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			lowerCAmelCase__       (  lowerCamelCase_   ,						unittest.TestCase					):
					lowerCAmelCase_					       =					None
					lowerCAmelCase_					       =					BloomTokenizerFast
					lowerCAmelCase_					       =					BloomTokenizerFast
					lowerCAmelCase_					       =					True
					lowerCAmelCase_					       =					False
					lowerCAmelCase_					       =					'''tokenizer_file'''
					lowerCAmelCase_					       =					{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}






					def      _snake_case   (       self		):


											"""simple docstring"""





											super().setUp()
											lowercase_     :					List[Any]													=	BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''		)
											tokenizer.save_pretrained(self.tmpdirname		)






					def      _snake_case   (       self    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											kwargs.update(self.special_tokens_map		)
											return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	self.get_rust_tokenizer()

											lowercase_     :					Optional[Any]													=	['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
											lowercase_     :					Union[str, Any]													=	[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

											lowercase_     :					Dict													=	tokenizer.batch_encode_plus(__SCREAMING_SNAKE_CASE		)['''input_ids''']
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Tuple													=	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE		)
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE=6		):


											"""simple docstring"""





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																							lowercase_     :					int													=	self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
																							# tokenizer_r.pad_token = None # Hotfixing padding = None
																							# Simple input
																							lowercase_     :					Optional[int]													=	'''This is a simple input'''
																							lowercase_     :					List[str]													=	['''This is a simple input 1''', '''This is a simple input 2''']
																							lowercase_     :					List[str]													=	('''This is a simple input''', '''This is a pair''')
																							lowercase_     :					Union[str, Any]													=	[
																							    ('''This is a simple input 1''', '''This is a simple input 2'''),
																							    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																							]

																							# Simple input tests
																							try:
																													tokenizer_r.encode(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)

																													tokenizer_r.batch_encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.encode(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.batch_encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																							except ValueError:
																													self.fail('''Bloom Tokenizer should be able to deal with padding'''		)

																							lowercase_     :					Optional[Any]													=	None  # Hotfixing padding = None
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Simple input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Simple input
																							self.assertRaises(
																							    __SCREAMING_SNAKE_CASE    ,							tokenizer_r.batch_encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''    ,							)

																							# Pair input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Pair input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Pair input
																							self.assertRaises(
																							    __SCREAMING_SNAKE_CASE    ,							tokenizer_r.batch_encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''    ,							)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[str]													=	self.get_rust_tokenizer()
											lowercase_     :					Tuple													=	load_dataset('''xnli'''    ,							'''all_languages'''    ,							split='''test'''    ,							streaming=__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[int]													=	next(iter(__SCREAMING_SNAKE_CASE		)		)['''premise''']  # pick up one data
											lowercase_     :					List[Any]													=	list(sample_data.values()		)

											lowercase_     :					Tuple													=	list(map(tokenizer.encode    ,							__SCREAMING_SNAKE_CASE		)		)
											lowercase_     :					Any													=	[tokenizer.decode(__SCREAMING_SNAKE_CASE    ,							clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE		) for x in output_tokens]
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map		)    ,							1		)
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()		)[0]		)    ,							1		)



