



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case     :List[str]			       =logging.get_logger(__name__)
__snake_case     :Dict			       ={'vocab_file': 'spiece.model'}

__snake_case     :int			       ={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__snake_case     :Union[str, Any]			       ={
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

__snake_case     :Optional[Any]			       ='▁'



class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     int								=     VOCAB_FILES_NAMES
				A_   :     int								=     PRETRAINED_VOCAB_FILES_MAP
				A_   :     Any								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


				def __init__(							self      :  Any						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  Union[str, Any]=True						, __UpperCamelCase      :  Any=True						, __UpperCamelCase      :  List[str]=False						, __UpperCamelCase      :  int="[CLS]"						, __UpperCamelCase      :  Any="[SEP]"						, __UpperCamelCase      :  Optional[Any]="<unk>"						, __UpperCamelCase      :  Optional[Any]="[SEP]"						, __UpperCamelCase      :  Dict="<pad>"						, __UpperCamelCase      :  Any="[CLS]"						, __UpperCamelCase      :  Optional[Any]="[MASK]"						, __UpperCamelCase      :  Optional[Dict[str, Any]] = None						, **__UpperCamelCase      :  Any						, )		->     None:
											# Mask token behave like a normal word, i.e. include the space before it and
											# is included in the raw text, there should be a match in a non-normalized sentence.
											A      						=    (
											    AddedToken(__UpperCamelCase						, lstrip=__UpperCamelCase						, rstrip=__UpperCamelCase						, normalized=__UpperCamelCase       )
											    if isinstance(__UpperCamelCase						, __UpperCamelCase       )
											    else mask_token
											)

											A      						=    {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=__UpperCamelCase						, remove_space=__UpperCamelCase						, keep_accents=__UpperCamelCase						, bos_token=__UpperCamelCase						, eos_token=__UpperCamelCase						, unk_token=__UpperCamelCase						, sep_token=__UpperCamelCase						, pad_token=__UpperCamelCase						, cls_token=__UpperCamelCase						, mask_token=__UpperCamelCase						, sp_model_kwargs=self.sp_model_kwargs						, **__UpperCamelCase						, )

											A      						=    do_lower_case
											A      						=    remove_space
											A      						=    keep_accents
											A      						=    vocab_file

											A      						=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(__UpperCamelCase       )


				@property
				def __UpperCamelCase						(							self      :  int       )		->     Union[str, Any]:
											return len(self.sp_model       )


				def __UpperCamelCase						(							self      :  Dict       )		->     List[Any]:
											A      						=    {self.convert_ids_to_tokens(__UpperCamelCase       ): i for i in range(self.vocab_size       )}
											vocab.update(self.added_tokens_encoder       )
											return vocab


				def __getstate__(							self      :  Any       )		->     List[str]:
											A      						=    self.__dict__.copy()
											A      						=    None
											return state


				def __setstate__(							self      :  Optional[Any]						, __UpperCamelCase      :  Tuple       )		->     int:
											A      						=    d

											# for backward compatibility
											if not hasattr(self						, 'sp_model_kwargs'       ):
																		A      						=    {}

											A      						=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(self.vocab_file       )


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  Union[str, Any]       )		->     Optional[int]:
											if self.remove_space:
																		A      						=    ' '.join(inputs.strip().split()       )
											else:
																		A      						=    inputs
											A      						=    outputs.replace('``'						, '"'       ).replace('\'\''						, '"'       )

											if not self.keep_accents:
																		A      						=    unicodedata.normalize('NFKD'						, __UpperCamelCase       )
																		A      						=    ''.join([c for c in outputs if not unicodedata.combining(__UpperCamelCase       )]       )
											if self.do_lower_case:
																		A      						=    outputs.lower()

											return outputs


				def __UpperCamelCase						(							self      :  Tuple						, __UpperCamelCase      :  str       )		->     List[str]:
											A      						=    self.preprocess_text(__UpperCamelCase       )
											A      						=    self.sp_model.encode(__UpperCamelCase						, out_type=__UpperCamelCase       )
											A      						=    []
											for piece in pieces:
																		if len(__UpperCamelCase       ) > 1 and piece[-1] == str(','       ) and piece[-2].isdigit():
																									A      						=    self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCamelCase						, ''       )       )
																									if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																if len(cur_pieces[0]       ) == 1:
																																							A      						=    cur_pieces[1:]
																																else:
																																							A      						=    cur_pieces[0][1:]
																									cur_pieces.append(piece[-1]       )
																									new_pieces.extend(__UpperCamelCase       )
																		else:
																									new_pieces.append(__UpperCamelCase       )

											return new_pieces


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Any       )		->     Optional[int]:
											return self.sp_model.PieceToId(__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[Any]						, __UpperCamelCase      :  Optional[int]       )		->     Optional[Any]:
											return self.sp_model.IdToPiece(__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  List[str]       )		->     List[str]:
											A      						=    []
											A      						=    ''
											A      						=    False
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									if not prev_is_special:
																																out_string += " "
																									out_string += self.sp_model.decode(__UpperCamelCase       ) + token
																									A      						=    True
																									A      						=    []
																		else:
																									current_sub_tokens.append(__UpperCamelCase       )
																									A      						=    False
											out_string += self.sp_model.decode(__UpperCamelCase       )
											return out_string.strip()


				def __UpperCamelCase						(							self      :  Dict						, __UpperCamelCase      :  List[int]						, __UpperCamelCase      :  Optional[List[int]] = None       )		->     List[int]:
											A      						=    [self.sep_token_id]
											A      						=    [self.cls_token_id]
											if token_ids_a is None:
																		return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep


				def __UpperCamelCase						(							self      :  Tuple						, __UpperCamelCase      :  List[int]						, __UpperCamelCase      :  Optional[List[int]] = None						, __UpperCamelCase      :  bool = False       )		->     List[int]:

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=__UpperCamelCase						, token_ids_a=__UpperCamelCase						, already_has_special_tokens=__UpperCamelCase       )

											if token_ids_a is not None:
																		return [1] + ([0] * len(__UpperCamelCase       )) + [1] + ([0] * len(__UpperCamelCase       )) + [1]
											return [1] + ([0] * len(__UpperCamelCase       )) + [1]


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  List[int]						, __UpperCamelCase      :  Optional[List[int]] = None       )		->     List[int]:
											A      						=    [self.sep_token_id]
											A      						=    [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[str] = None       )		->     Tuple[str]:
											if not os.path.isdir(__UpperCamelCase       ):
																		logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
																		return
											A      						=    os.path.join(
											    __UpperCamelCase						, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

											if os.path.abspath(self.vocab_file       ) != os.path.abspath(__UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
																		copyfile(self.vocab_file						, __UpperCamelCase       )
											elif not os.path.isfile(self.vocab_file       ):
																		with open(__UpperCamelCase						, 'wb'       ) as fi:
																									A      						=    self.sp_model.serialized_model_proto()
																									fi.write(__UpperCamelCase       )

											return (out_vocab_file,)




import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     LEDTokenizer
				A_   :     Union[str, Any]								=     LEDTokenizerFast
				A_   :     Tuple								=     True


				def __UpperCamelCase						(							self      :  str       )		->     str:
											super().setUp()
											A      						=    [
											    'l',
											    'o',
											    'w',
											    'e',
											    'r',
											    's',
											    't',
											    'i',
											    'd',
											    'n',
											    '\u0120',
											    '\u0120l',
											    '\u0120n',
											    '\u0120lo',
											    '\u0120low',
											    'er',
											    '\u0120lowest',
											    '\u0120newer',
											    '\u0120wider',
											    '<unk>',
											]
											A      						=    dict(zip(__UpperCamelCase						, range(len(__UpperCamelCase       )       )       )       )
											A      						=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as fp:
																		fp.write(json.dumps(__UpperCamelCase       ) + '\n'       )
											with open(self.merges_file						, 'w'						, encoding='utf-8'       ) as fp:
																		fp.write('\n'.join(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  str						, **__UpperCamelCase      :  str       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return self.tokenizer_class.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]						, **__UpperCamelCase      :  List[Any]       )		->     str:
											kwargs.update(self.special_tokens_map       )
											return self.rust_tokenizer_class.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[Any]						, __UpperCamelCase      :  Optional[int]       )		->     Dict:
											return "lower newer", "lower newer"


				@cached_property
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Any:
											return LEDTokenizer.from_pretrained('allenai/led-base-16384'       )


				@cached_property
				def __UpperCamelCase						(							self      :  str       )		->     str:
											return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'       )


				@require_torch
				def __UpperCamelCase						(							self      :  str       )		->     Optional[Any]:
											A      						=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
											A      						=    [0, 250, 251, 17_818, 13, 39_186, 1_938, 4, 2]

											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    tokenizer(__UpperCamelCase						, max_length=len(__UpperCamelCase       )						, padding=__UpperCamelCase						, return_tensors='pt'       )
																		self.assertIsInstance(__UpperCamelCase						, __UpperCamelCase       )

																		self.assertEqual((2, 9)						, batch.input_ids.shape       )
																		self.assertEqual((2, 9)						, batch.attention_mask.shape       )
																		A      						=    batch.input_ids.tolist()[0]
																		self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@require_torch
				def __UpperCamelCase						(							self      :  Optional[int]       )		->     str:
											A      						=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase						, return_tensors='pt'       )
																		self.assertIn('input_ids'						, __UpperCamelCase       )
																		self.assertIn('attention_mask'						, __UpperCamelCase       )
																		self.assertNotIn('labels'						, __UpperCamelCase       )
																		self.assertNotIn('decoder_attention_mask'						, __UpperCamelCase       )


				@require_torch
				def __UpperCamelCase						(							self      :  List[Any]       )		->     Union[str, Any]:
											A      						=    [
											    'Summary of the text.',
											    'Another summary.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    tokenizer(text_target=__UpperCamelCase						, max_length=32						, padding='max_length'						, return_tensors='pt'       )
																		self.assertEqual(32						, targets['input_ids'].shape[1]       )


				@require_torch
				def __UpperCamelCase						(							self      :  List[str]       )		->     Any:
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    tokenizer(
																		    ['I am a small frog' * 1_024, 'I am a small frog']						, padding=__UpperCamelCase						, truncation=__UpperCamelCase						, return_tensors='pt'       )
																		self.assertIsInstance(__UpperCamelCase						, __UpperCamelCase       )
																		self.assertEqual(batch.input_ids.shape						, (2, 5_122)       )


				@require_torch
				def __UpperCamelCase						(							self      :  int       )		->     Optional[Any]:
											A      						=    ['A long paragraph for summarization.']
											A      						=    [
											    'Summary of the text.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    tokenizer(__UpperCamelCase						, return_tensors='pt'       )
																		A      						=    tokenizer(text_target=__UpperCamelCase						, return_tensors='pt'       )
																		A      						=    inputs['input_ids']
																		A      						=    targets['input_ids']
																		self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()       )
																		self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()       )
																		self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()       )
																		self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()       )


				@require_torch
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Tuple:
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		A      						=    ['Summary of the text.', 'Another summary.']
																		A      						=    [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

																		A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
																		A      						=    [[0] * len(__UpperCamelCase       ) for x in encoded_output['input_ids']]
																		A      						=    tokenizer.pad(__UpperCamelCase       )
																		self.assertSequenceEqual(outputs['global_attention_mask']						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str       )		->     int:
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     Any:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																									A      						=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase						, **__UpperCamelCase       )
																									A      						=    self.tokenizer_class.from_pretrained(__UpperCamelCase						, **__UpperCamelCase       )
																									A      						=    'A, <mask> AllenNLP sentence.'
																									A      						=    tokenizer_r.encode_plus(__UpperCamelCase						, add_special_tokens=__UpperCamelCase						, return_token_type_ids=__UpperCamelCase       )
																									A      						=    tokenizer_p.encode_plus(__UpperCamelCase						, add_special_tokens=__UpperCamelCase						, return_token_type_ids=__UpperCamelCase       )
																									self.assertEqual(sum(tokens_r['token_type_ids']       )						, sum(tokens_p['token_type_ids']       )       )
																									self.assertEqual(
																									    sum(tokens_r['attention_mask']       ) / len(tokens_r['attention_mask']       )						, sum(tokens_p['attention_mask']       ) / len(tokens_p['attention_mask']       )						, )

																									A      						=    tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']       )
																									A      						=    tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']       )
																									self.assertSequenceEqual(tokens_p['input_ids']						, [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]       )
																									self.assertSequenceEqual(tokens_r['input_ids']						, [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]       )

																									self.assertSequenceEqual(
																									    __UpperCamelCase						, ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']       )
																									self.assertSequenceEqual(
																									    __UpperCamelCase						, ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']       )