





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__											= logging.get_logger(__name__)

UpperCamelCase__											= {"vocab_file": "spiece.model"}

UpperCamelCase__											= {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

UpperCamelCase__											= {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
UpperCamelCase__											= 0
UpperCamelCase__											= 1
UpperCamelCase__											= 2
UpperCamelCase__											= 3
UpperCamelCase__											= 4

class        __SCREAMING_SNAKE_CASE					(     _a       ):
							snake_case     :      Tuple						     =       VOCAB_FILES_NAMES
							snake_case     :      Any						     =       PRETRAINED_VOCAB_FILES_MAP
							snake_case     :      Tuple						     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case     :      Tuple						     =       """left"""






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase=False  , __lowerCAmelCase=True  , __lowerCAmelCase=False  , __lowerCAmelCase="<s>"  , __lowerCAmelCase="</s>"  , __lowerCAmelCase="<unk>"  , __lowerCAmelCase="<sep>"  , __lowerCAmelCase="<pad>"  , __lowerCAmelCase="<cls>"  , __lowerCAmelCase="<mask>"  , __lowerCAmelCase=["<eop>", "<eod>"]  , __lowerCAmelCase = None  , **__lowerCAmelCase  , ):
														# Mask token behave like a normal word, i.e. include the space before it
														UpperCamelCase__				     =			AddedToken(__lowerCAmelCase  , lstrip=__lowerCAmelCase  , rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase  , __lowerCAmelCase  ) else mask_token

														UpperCamelCase__				     =			{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=__lowerCAmelCase  , remove_space=__lowerCAmelCase  , keep_accents=__lowerCAmelCase  , bos_token=__lowerCAmelCase  , eos_token=__lowerCAmelCase  , unk_token=__lowerCAmelCase  , sep_token=__lowerCAmelCase  , pad_token=__lowerCAmelCase  , cls_token=__lowerCAmelCase  , mask_token=__lowerCAmelCase  , additional_special_tokens=__lowerCAmelCase  , sp_model_kwargs=self.sp_model_kwargs  , **__lowerCAmelCase  , )

														UpperCamelCase__				     =			3

														UpperCamelCase__				     =			do_lower_case
														UpperCamelCase__				     =			remove_space
														UpperCamelCase__				     =			keep_accents
														UpperCamelCase__				     =			vocab_file

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(__lowerCAmelCase  )






							@property
							def 				_lowerCamelCase		(						self  ):
														return len(self.sp_model  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			{self.convert_ids_to_tokens(__lowerCAmelCase  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab






							def __getstate__(						self  ):
														UpperCamelCase__				     =			self.__dict__.copy()
														UpperCamelCase__				     =			None
														return state






							def __setstate__(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			d

														# for backward compatibility
														if not hasattr(self  , """sp_model_kwargs"""  ):
																					UpperCamelCase__				     =			{}

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if self.remove_space:
																					UpperCamelCase__				     =			""" """.join(inputs.strip().split()  )
														else:
																					UpperCamelCase__				     =			inputs
														UpperCamelCase__				     =			outputs.replace("""``"""  , """\""""  ).replace("""''"""  , """\""""  )

														if not self.keep_accents:
																					UpperCamelCase__				     =			unicodedata.normalize("""NFKD"""  , __lowerCAmelCase  )
																					UpperCamelCase__				     =			"""""".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase  )]  )
														if self.do_lower_case:
																					UpperCamelCase__				     =			outputs.lower()

														return outputs






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.preprocess_text(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.sp_model.encode(__lowerCAmelCase  , out_type=__lowerCAmelCase  )
														UpperCamelCase__				     =			[]
														for piece in pieces:
																					if len(__lowerCAmelCase  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
																												UpperCamelCase__				     =			self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase  , """"""  )  )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]  ) == 1:
																																										UpperCamelCase__				     =			cur_pieces[1:]
																																			else:
																																										UpperCamelCase__				     =			cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]  )
																												new_pieces.extend(__lowerCAmelCase  )
																					else:
																												new_pieces.append(__lowerCAmelCase  )

														return new_pieces






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.PieceToId(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.IdToPiece(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			"""""".join(__lowerCAmelCase  ).replace(__lowerCAmelCase  , """ """  ).strip()
														return out_string






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = False  , __lowerCAmelCase = None  , __lowerCAmelCase = True  , **__lowerCAmelCase  , ):
														UpperCamelCase__				     =			kwargs.pop("""use_source_tokenizer"""  , __lowerCAmelCase  )

														UpperCamelCase__				     =			self.convert_ids_to_tokens(__lowerCAmelCase  , skip_special_tokens=__lowerCAmelCase  )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			[]
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase  )  )
																																			UpperCamelCase__				     =			[]
																												sub_texts.append(__lowerCAmelCase  )
																					else:
																												current_sub_text.append(__lowerCAmelCase  )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase  )  )

														# Mimic the behavior of the Rust tokenizer:
														# By default, there are no spaces between special tokens
														UpperCamelCase__				     =			"""""".join(__lowerCAmelCase  )

														UpperCamelCase__				     =			(
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					UpperCamelCase__				     =			self.clean_up_tokenization(__lowerCAmelCase  )
																					return clean_text
														else:
																					return text






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  , __lowerCAmelCase = False  ):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase  , token_ids_a=__lowerCAmelCase  , already_has_special_tokens=__lowerCAmelCase  )

														if token_ids_a is not None:
																					return ([0] * len(__lowerCAmelCase  )) + [1] + ([0] * len(__lowerCAmelCase  )) + [1, 1]
														return ([0] * len(__lowerCAmelCase  )) + [1, 1]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[2]

														if token_ids_a is None:
																					return len(token_ids_a + sep  ) * [0] + cls_segment_id
														return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														if not os.path.isdir(__lowerCAmelCase  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														UpperCamelCase__				     =			os.path.join(
														    __lowerCAmelCase  , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCAmelCase  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  , __lowerCAmelCase  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(__lowerCAmelCase  , """wb"""  ) as fi:
																												UpperCamelCase__				     =			self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase  )

														return (out_vocab_file,)











from __future__ import annotations

import unittest

from transformers import AutoTokenizer, PegasusConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFAutoModelForSeqaSeqLM, TFPegasusForConditionalGeneration, TFPegasusModel

@require_tf
class        __SCREAMING_SNAKE_CASE					:
							snake_case     :      Dict						     =       PegasusConfig
							snake_case     :      Any						     =       {}
							snake_case     :      int						     =       """gelu"""






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase=13  , __lowerCAmelCase=7  , __lowerCAmelCase=True  , __lowerCAmelCase=False  , __lowerCAmelCase=99  , __lowerCAmelCase=32  , __lowerCAmelCase=2  , __lowerCAmelCase=4  , __lowerCAmelCase=37  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.1  , __lowerCAmelCase=40  , __lowerCAmelCase=2  , __lowerCAmelCase=1  , __lowerCAmelCase=0  , ):
														UpperCamelCase__				     =			parent
														UpperCamelCase__				     =			batch_size
														UpperCamelCase__				     =			seq_length
														UpperCamelCase__				     =			is_training
														UpperCamelCase__				     =			use_labels
														UpperCamelCase__				     =			vocab_size
														UpperCamelCase__				     =			hidden_size
														UpperCamelCase__				     =			num_hidden_layers
														UpperCamelCase__				     =			num_attention_heads
														UpperCamelCase__				     =			intermediate_size

														UpperCamelCase__				     =			hidden_dropout_prob
														UpperCamelCase__				     =			attention_probs_dropout_prob
														UpperCamelCase__				     =			max_position_embeddings
														UpperCamelCase__				     =			eos_token_id
														UpperCamelCase__				     =			pad_token_id
														UpperCamelCase__				     =			bos_token_id






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length - 1]  , self.vocab_size  )
														UpperCamelCase__				     =			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size  )  , 1  )
														UpperCamelCase__				     =			tf.concat([input_ids, eos_tensor]  , axis=1  )

														UpperCamelCase__				     =			ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size  )

														UpperCamelCase__				     =			self.config_cls(
														    vocab_size=self.vocab_size  , d_model=self.hidden_size  , encoder_layers=self.num_hidden_layers  , decoder_layers=self.num_hidden_layers  , encoder_attention_heads=self.num_attention_heads  , decoder_attention_heads=self.num_attention_heads  , encoder_ffn_dim=self.intermediate_size  , decoder_ffn_dim=self.intermediate_size  , dropout=self.hidden_dropout_prob  , attention_dropout=self.attention_probs_dropout_prob  , max_position_embeddings=self.max_position_embeddings  , eos_token_ids=[2]  , bos_token_id=self.bos_token_id  , pad_token_id=self.pad_token_id  , decoder_start_token_id=self.pad_token_id  , **self.config_updates  , )
														UpperCamelCase__				     =			prepare_pegasus_inputs_dict(__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
														return config, inputs_dict







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			TFPegasusModel(config=__lowerCAmelCase  ).get_decoder()
														UpperCamelCase__				     =			inputs_dict["""input_ids"""]

														UpperCamelCase__				     =			input_ids[:1, :]
														UpperCamelCase__				     =			inputs_dict["""attention_mask"""][:1, :]
														UpperCamelCase__				     =			inputs_dict["""head_mask"""]
														UpperCamelCase__				     =			1

														# first forward pass
														UpperCamelCase__				     =			model(__lowerCAmelCase  , attention_mask=__lowerCAmelCase  , head_mask=__lowerCAmelCase  , use_cache=__lowerCAmelCase  )

														UpperCamelCase__     ,     UpperCamelCase__				     =			outputs.to_tuple()

														# create hypothetical next token and extent to next_input_ids
														UpperCamelCase__				     =			ids_tensor((self.batch_size, 3)  , config.vocab_size  )
														UpperCamelCase__				     =			tf.cast(ids_tensor((self.batch_size, 3)  , 2  )  , tf.inta  )

														# append to next input_ids and
														UpperCamelCase__				     =			tf.concat([input_ids, next_tokens]  , axis=-1  )
														UpperCamelCase__				     =			tf.concat([attention_mask, next_attn_mask]  , axis=-1  )

														UpperCamelCase__				     =			model(__lowerCAmelCase  , attention_mask=__lowerCAmelCase  )[0]
														UpperCamelCase__				     =			model(__lowerCAmelCase  , attention_mask=__lowerCAmelCase  , past_key_values=__lowerCAmelCase  )[0]

														self.parent.assertEqual(next_tokens.shape[1]  , output_from_past.shape[1]  )

														# select random slice
														UpperCamelCase__				     =			int(ids_tensor((1,)  , output_from_past.shape[-1]  )  )
														UpperCamelCase__				     =			output_from_no_past[:, -3:, random_slice_idx]
														UpperCamelCase__				     =			output_from_past[:, :, random_slice_idx]

														# test that outputs are equal for slice
														tf.debugging.assert_near(__lowerCAmelCase  , __lowerCAmelCase  , rtol=1E-3  )






def     _UpperCamelCase     (a__   :List[str]   ,			a__   :Any   ,			a__   :str   ,			a__   :Optional[int]=None   ,			a__   :Union[str, Any]=None   ,			a__   :Optional[int]=None   ,			a__   :Optional[int]=None   ,			a__   :List[str]=None   ,			):






							"""simple docstring"""





							if attention_mask is None:
														UpperCamelCase__				     =			tf.cast(tf.math.not_equal(a__   ,			config.pad_token_id       )   ,			tf.inta       )
							if decoder_attention_mask is None:
														UpperCamelCase__				     =			tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape   ,			dtype=tf.inta       ),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]   ,			config.pad_token_id       )   ,			tf.inta       ),
														    ]   ,			axis=-1   ,			)
							if head_mask is None:
														UpperCamelCase__				     =			tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
							if decoder_head_mask is None:
														UpperCamelCase__				     =			tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
							if cross_attn_head_mask is None:
														UpperCamelCase__				     =			tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}

@require_tf
class        __SCREAMING_SNAKE_CASE					(     _a   ,		_a   ,		unittest.TestCase       ):
							snake_case     :      Optional[int]						     =       (TFPegasusForConditionalGeneration, TFPegasusModel) if is_tf_available() else ()
							snake_case     :      Dict						     =       (TFPegasusForConditionalGeneration,) if is_tf_available() else ()
							snake_case     :      Optional[int]						     =       (
							    {
							        """conversational""": TFPegasusForConditionalGeneration,
							        """feature-extraction""": TFPegasusModel,
							        """summarization""": TFPegasusForConditionalGeneration,
							        """text2text-generation""": TFPegasusForConditionalGeneration,
							        """translation""": TFPegasusForConditionalGeneration,
							    }
							    if is_tf_available()
							    else {}
							)
							snake_case     :      Optional[Any]						     =       True
							snake_case     :      int						     =       False
							snake_case     :      int						     =       False






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			TFPegasusModelTester(self  )
														UpperCamelCase__				     =			ConfigTester(self  , config_class=__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														self.config_tester.run_common_tests()







							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self.model_tester.prepare_config_and_inputs_for_common()
														self.model_tester.check_decoder_model_past_large_inputs(*__lowerCAmelCase  )






@require_sentencepiece
@require_tokenizers
@require_tf
class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):
							snake_case     :      str						     =       [
							    """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""",
							    """ The London trio are up for best UK act and best album, as well as getting two nominations in the best song category.\"We got told like this morning 'Oh I think you're nominated'\", said Dappy.\"And I was like 'Oh yeah, which one?' And now we've got nominated for four awards. I mean, wow!\"Bandmate Fazer added: \"We thought it's best of us to come down and mingle with everyone and say hello to the cameras. And now we find we've got four nominations.\"The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn't be too disappointed if they didn't win this time around.\"At the end of the day we're grateful to be where we are in our careers.\"If it don't happen then it don't happen - live to fight another day and keep on making albums and hits for the fans.\"Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers' All These Things That I've Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year's Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border.\"We just done Edinburgh the other day,\" said Dappy.\"We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!\" """,
							]
							snake_case     :      List[Any]						     =       [
							    """California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to"""
							    """ reduce the risk of wildfires.""",
							    """N-Dubz have revealed they\'re \"grateful\" to have been nominated for four Mobo Awards.""",
							]  # differs slightly from pytorch, likely due to numerical differences in linear layers
							snake_case     :      Dict						     =       """google/pegasus-xsum"""






							@cached_property
							def 				_lowerCamelCase		(						self  ):
														return AutoTokenizer.from_pretrained(self.model_name  )






							@cached_property
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name  )
														return model






							def 				_lowerCamelCase		(						self  , **__lowerCAmelCase  ):
														UpperCamelCase__				     =			self.translate_src_text(**__lowerCAmelCase  )
														assert self.expected_text == generated_words






							def 				_lowerCamelCase		(						self  , **__lowerCAmelCase  ):
														UpperCamelCase__				     =			self.tokenizer(self.src_text  , **__lowerCAmelCase  , padding=__lowerCAmelCase  , return_tensors="""tf"""  )
														UpperCamelCase__				     =			self.model.generate(
														    model_inputs.input_ids  , attention_mask=model_inputs.attention_mask  , num_beams=2  , use_cache=__lowerCAmelCase  , )
														UpperCamelCase__				     =			self.tokenizer.batch_decode(generated_ids.numpy()  , skip_special_tokens=__lowerCAmelCase  )
														return generated_words







							@slow
							def 				_lowerCamelCase		(						self  ):
														self._assert_generated_batch_equal_expected()




