
'''simple docstring'''



import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    MODEL_FOR_PRETRAINING_MAPPING,
						    AlbertForMaskedLM,
						    AlbertForMultipleChoice,
						    AlbertForPreTraining,
						    AlbertForQuestionAnswering,
						    AlbertForSequenceClassification,
						    AlbertForTokenClassification,
						    AlbertModel,
						)
						from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class     UpperCAmelCase_	:






						'''simple docstring'''






						def __init__(   self       ,  _lowercase       ,  _lowercase=13       ,  _lowercase=7       ,  _lowercase=True       ,  _lowercase=True       ,  _lowercase=True       ,  _lowercase=True       ,  _lowercase=99       ,  _lowercase=16       ,  _lowercase=36       ,  _lowercase=6       ,  _lowercase=6       ,  _lowercase=6       ,  _lowercase=37       ,  _lowercase="gelu"       ,  _lowercase=0.1       ,  _lowercase=0.1       ,  _lowercase=512       ,  _lowercase=16       ,  _lowercase=2       ,  _lowercase=0.02       ,  _lowercase=3       ,  _lowercase=4       ,  _lowercase=None       ,  ):

												"""simple docstring"""



												_lowerCAmelCase    		=					parent
												_lowerCAmelCase    		=					batch_size
												_lowerCAmelCase    		=					seq_length
												_lowerCAmelCase    		=					is_training
												_lowerCAmelCase    		=					use_input_mask
												_lowerCAmelCase    		=					use_token_type_ids
												_lowerCAmelCase    		=					use_labels
												_lowerCAmelCase    		=					vocab_size
												_lowerCAmelCase    		=					embedding_size
												_lowerCAmelCase    		=					hidden_size
												_lowerCAmelCase    		=					num_hidden_layers
												_lowerCAmelCase    		=					num_hidden_groups
												_lowerCAmelCase    		=					num_attention_heads
												_lowerCAmelCase    		=					intermediate_size
												_lowerCAmelCase    		=					hidden_act
												_lowerCAmelCase    		=					hidden_dropout_prob
												_lowerCAmelCase    		=					attention_probs_dropout_prob
												_lowerCAmelCase    		=					max_position_embeddings
												_lowerCAmelCase    		=					type_vocab_size
												_lowerCAmelCase    		=					type_sequence_label_size
												_lowerCAmelCase    		=					initializer_range
												_lowerCAmelCase    		=					num_labels
												_lowerCAmelCase    		=					num_choices
												_lowerCAmelCase    		=					scope




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size )

												_lowerCAmelCase    		=					None
												if self.use_input_mask:
																		_lowerCAmelCase    		=					random_attention_mask([self.batch_size, self.seq_length] )

												_lowerCAmelCase    		=					None
												if self.use_token_type_ids:
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size )

												_lowerCAmelCase    		=					None
												_lowerCAmelCase    		=					None
												_lowerCAmelCase    		=					None
												if self.use_labels:
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size]       ,  self.type_sequence_label_size )
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels )
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size]       ,  self.num_choices )

												_lowerCAmelCase    		=					self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




						def    _lowercase      (   self ):

												"""simple docstring"""



												return AlbertConfig(
												    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  initializer_range=self.initializer_range       ,  num_hidden_groups=self.num_hidden_groups       ,  )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertModel(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase )
												_lowerCAmelCase    		=					model(_lowercase       ,  token_type_ids=_lowercase )
												_lowerCAmelCase    		=					model(_lowercase )
												self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size) )
												self.parent.assertEqual(result.pooler_output.shape       ,  (self.batch_size, self.hidden_size) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertForPreTraining(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(
												    _lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  labels=_lowercase       ,  sentence_order_label=_lowercase       ,  )
												self.parent.assertEqual(result.prediction_logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size) )
												self.parent.assertEqual(result.sop_logits.shape       ,  (self.batch_size, config.num_labels) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertForMaskedLM(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertForQuestionAnswering(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(
												    _lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  start_positions=_lowercase       ,  end_positions=_lowercase       ,  )
												self.parent.assertEqual(result.start_logits.shape       ,  (self.batch_size, self.seq_length) )
												self.parent.assertEqual(result.end_logits.shape       ,  (self.batch_size, self.seq_length) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.num_labels
												_lowerCAmelCase    		=					AlbertForSequenceClassification(_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_labels) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.num_labels
												_lowerCAmelCase    		=					AlbertForTokenClassification(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.num_labels) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.num_choices
												_lowerCAmelCase    		=					AlbertForMultipleChoice(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					input_ids.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												_lowerCAmelCase    		=					token_type_ids.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												_lowerCAmelCase    		=					input_mask.unsqueeze(1 ).expand(-1       ,  self.num_choices       ,  -1 ).contiguous()
												_lowerCAmelCase    		=					model(
												    _lowercase       ,  attention_mask=_lowercase       ,  token_type_ids=_lowercase       ,  labels=_lowercase       ,  )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.num_choices) )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.prepare_config_and_inputs()
												(
												    (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       (
												    _lowerCAmelCase
												)		,       
												)    		=					config_and_inputs
												_lowerCAmelCase    		=					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict






@require_torch
class     UpperCAmelCase_	(						_SCREAMING_SNAKE_CASE			,				_SCREAMING_SNAKE_CASE			,				unittest.TestCase					):






						'''simple docstring'''






						_lowercase						: Optional[int]										=  (
						    (
						        AlbertModel,
						        AlbertForPreTraining,
						        AlbertForMaskedLM,
						        AlbertForMultipleChoice,
						        AlbertForSequenceClassification,
						        AlbertForTokenClassification,
						        AlbertForQuestionAnswering,
						    )
						    if is_torch_available()
						    else ()
						)
						_lowercase						: Union[str, Any]										=  (
						    {
						        '''feature-extraction''': AlbertModel,
						        '''fill-mask''': AlbertForMaskedLM,
						        '''question-answering''': AlbertForQuestionAnswering,
						        '''text-classification''': AlbertForSequenceClassification,
						        '''token-classification''': AlbertForTokenClassification,
						        '''zero-shot''': AlbertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_lowercase						: Any										=  True




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase=False ):

												"""simple docstring"""



												_lowerCAmelCase    		=					super()._prepare_for_class(_lowercase       ,  _lowercase       ,  return_labels=_lowercase )

												if return_labels:
																		if model_class in get_values(_lowercase ):
																								_lowerCAmelCase    		=					torch.zeros(
																								    (self.model_tester.batch_size, self.model_tester.seq_length)       ,  dtype=torch.long       ,  device=_lowercase )
																								_lowerCAmelCase    		=					torch.zeros(
																								    self.model_tester.batch_size       ,  dtype=torch.long       ,  device=_lowercase )
												return inputs_dict




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertModelTester(self )
												_lowerCAmelCase    		=					ConfigTester(self       ,  config_class=_lowercase       ,  hidden_size=37 )




						def    _lowercase      (   self ):

												"""simple docstring"""



												self.config_tester.run_common_tests()




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_pretraining(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																		_lowerCAmelCase    		=					type
																		self.model_tester.create_and_check_model(*_lowercase )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		_lowerCAmelCase    		=					AlbertModel.from_pretrained(_lowercase )
																		self.assertIsNotNone(_lowercase )






@require_torch
class     UpperCAmelCase_	(						unittest.TestCase					):






						'''simple docstring'''






						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					AlbertModel.from_pretrained("""albert-base-v2""" )
												_lowerCAmelCase    		=					torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]] )
												_lowerCAmelCase    		=					torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(_lowercase       ,  attention_mask=_lowercase )[0]
												_lowerCAmelCase    		=					torch.Size((1, 11, 768) )
												self.assertEqual(output.shape       ,  _lowercase )
												_lowerCAmelCase    		=					torch.tensor(
												    [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]] )

												self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,  _lowercase       ,  atol=1e-4 ) )




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A				:       List[str]      =  logging.get_logger(__name__)

A				:       List[Any]      =  {'vocab_file': 'spiece.model'}

A				:       Tuple      =  {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

A				:       Any      =  {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
A				:       Tuple      =  0
A				:       str      =  1
A				:       str      =  2
A				:       Union[str, Any]      =  3
A				:       Optional[Any]      =  4






class      UpperCamelCase(	_a ):
					snake_case_		:					Union[str, Any]    					=						VOCAB_FILES_NAMES
					snake_case_		:					List[Any]    					=						PRETRAINED_VOCAB_FILES_MAP
					snake_case_		:					str    					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					snake_case_		:					Tuple    					=						"""left"""


					def __init__(    self						:		Optional[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		Any=False  ,     SCREAMING_SNAKE_CASE						:		List[Any]=True  ,     SCREAMING_SNAKE_CASE						:		Dict=False  ,     SCREAMING_SNAKE_CASE						:		Tuple="<s>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="</s>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<unk>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<sep>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"  ,     SCREAMING_SNAKE_CASE						:		Union[str, Any]="<cls>"  ,     SCREAMING_SNAKE_CASE						:		str="<mask>"  ,     SCREAMING_SNAKE_CASE						:		int=["<eop>", "<eod>"]  ,     SCREAMING_SNAKE_CASE						:		Optional[Dict[str, Any]] = None  ,     **SCREAMING_SNAKE_CASE						:		Dict  ,     )      ->						None:




											'''simple docstring'''

											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else mask_token

											__snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=SCREAMING_SNAKE_CASE  ,     remove_space=SCREAMING_SNAKE_CASE  ,     keep_accents=SCREAMING_SNAKE_CASE  ,     bos_token=SCREAMING_SNAKE_CASE  ,     eos_token=SCREAMING_SNAKE_CASE  ,     unk_token=SCREAMING_SNAKE_CASE  ,     sep_token=SCREAMING_SNAKE_CASE  ,     pad_token=SCREAMING_SNAKE_CASE  ,     cls_token=SCREAMING_SNAKE_CASE  ,     mask_token=SCREAMING_SNAKE_CASE  ,     additional_special_tokens=SCREAMING_SNAKE_CASE  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **SCREAMING_SNAKE_CASE  ,     )

											__snake_case				=		3

											__snake_case				=		do_lower_case
											__snake_case				=		remove_space
											__snake_case				=		keep_accents
											__snake_case				=		vocab_file

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(SCREAMING_SNAKE_CASE    )


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]    )      ->						Union[str, Any]:




											'''simple docstring'''

											return len(self.sp_model    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab


					def __getstate__(    self						:		Any    )      ->						str:




											'''simple docstring'''

											__snake_case				=		self.__dict__.copy()
											__snake_case				=		None
											return state


					def __setstate__(    self						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						int:




											'''simple docstring'''

											__snake_case				=		d

											# for backward compatibility
											if not hasattr(self  ,     "sp_model_kwargs"    ):
																	__snake_case				=		{}

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						int:




											'''simple docstring'''

											if self.remove_space:
																	__snake_case				=		" ".join(inputs.strip().split()    )
											else:
																	__snake_case				=		inputs
											__snake_case				=		outputs.replace("``"  ,     "\""    ).replace("''"  ,     "\""    )

											if not self.keep_accents:
																	__snake_case				=		unicodedata.normalize("NFKD"  ,     SCREAMING_SNAKE_CASE    )
																	__snake_case				=		"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE    )]    )
											if self.do_lower_case:
																	__snake_case				=		outputs.lower()

											return outputs


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		self.preprocess_text(SCREAMING_SNAKE_CASE    )
											__snake_case				=		self.sp_model.encode(SCREAMING_SNAKE_CASE  ,     out_type=SCREAMING_SNAKE_CASE    )
											__snake_case				=		[]
											for piece in pieces:
																	if len(SCREAMING_SNAKE_CASE    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
																							__snake_case				=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE  ,     ""    )    )
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]    ) == 1:
																																			__snake_case				=		cur_pieces[1:]
																													else:
																																			__snake_case				=		cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]    )
																							new_pieces.extend(SCREAMING_SNAKE_CASE    )
																	else:
																							new_pieces.append(SCREAMING_SNAKE_CASE    )

											return new_pieces


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						Optional[Any]:




											'''simple docstring'''

											return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[Any]    )      ->						Optional[int]:




											'''simple docstring'''

											return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		int    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    ).replace(SCREAMING_SNAKE_CASE  ,     " "    ).strip()
											return out_string


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		bool = False  ,     SCREAMING_SNAKE_CASE						:		bool = None  ,     SCREAMING_SNAKE_CASE						:		bool = True  ,     **SCREAMING_SNAKE_CASE						:		Optional[int]  ,     )      ->						str:




											'''simple docstring'''

											__snake_case				=		kwargs.pop("use_source_tokenizer"  ,     SCREAMING_SNAKE_CASE    )

											__snake_case				=		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE  ,     skip_special_tokens=SCREAMING_SNAKE_CASE    )

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__snake_case				=		[]
											__snake_case				=		[]
											for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																							continue
																	if token in self.added_tokens_encoder:
																							if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )
																													__snake_case				=		[]
																							sub_texts.append(SCREAMING_SNAKE_CASE    )
																	else:
																							current_sub_text.append(SCREAMING_SNAKE_CASE    )
											if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )

											# Mimic the behavior of the Rust tokenizer:
											# By default, there are no spaces between special tokens
											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    )

											__snake_case				=		(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
																	__snake_case				=		self.clean_up_tokenization(SCREAMING_SNAKE_CASE    )
																	return clean_text
											else:
																	return text


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											if token_ids_a is None:
																	return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None  ,     SCREAMING_SNAKE_CASE						:		bool = False    )      ->						List[int]:




											'''simple docstring'''

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE  ,     token_ids_a=SCREAMING_SNAKE_CASE  ,     already_has_special_tokens=SCREAMING_SNAKE_CASE    )

											if token_ids_a is not None:
																	return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]
											return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[2]

											if token_ids_a is None:
																	return len(token_ids_a + sep    ) * [0] + cls_segment_id
											return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id






					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		Optional[str] = None    )      ->						Tuple[str]:




											'''simple docstring'''

											if not os.path.isdir(SCREAMING_SNAKE_CASE    ):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
																	return
											__snake_case				=		os.path.join(
											    SCREAMING_SNAKE_CASE  ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE    ) and os.path.isfile(self.vocab_file    ):
																	copyfile(self.vocab_file  ,     SCREAMING_SNAKE_CASE    )
											elif not os.path.isfile(self.vocab_file    ):
																	with open(SCREAMING_SNAKE_CASE  ,     "wb"    ) as fi:
																							__snake_case				=		self.sp_model.serialized_model_proto()
																							fi.write(SCREAMING_SNAKE_CASE    )

											return (out_vocab_file,)





