
"""simple docstring"""
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__									=					get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
			from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__magic_name__									=					25_60_47
__magic_name__									=					25_61_45


@require_sentencepiece
@require_tokenizers
class  _lowerCAmelCase    (    lowerCamelCase     ,							unittest.TestCase    ):
							lowercase_  :  str      =				NllbTokenizer
							lowercase_  :  Optional[int]      =				NllbTokenizerFast
							lowercase_  :  Tuple      =				True
							lowercase_  :  Optional[int]      =				True
							lowercase_  :  List[Any]      =				{}



							def    _a (   self )				->				Tuple:
								super().setUp()

								# We have a SentencePiece fixture for testing
								_UpperCAmelCase			=    NllbTokenizer(SCREAMING_SNAKE_CASE_			,    keep_accents=SCREAMING_SNAKE_CASE_ )
								tokenizer.save_pretrained(self.tmpdirname )



							def    _a (   self )				->				List[Any]:
								_UpperCAmelCase			=    NllbTokenizer(SCREAMING_SNAKE_CASE_			,    keep_accents=SCREAMING_SNAKE_CASE_ )

								_UpperCAmelCase			=    tokenizer.tokenize("This is a test" )
								self.assertListEqual(SCREAMING_SNAKE_CASE_			,    ["▁This", "▁is", "▁a", "▁t", "est"] )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )			,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,    )

								_UpperCAmelCase			=    tokenizer.tokenize("I was born in 92000, and this is falsé." )
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE_			,    [
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "9",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "é",
								        ".",
								    ]			,    )
								_UpperCAmelCase			=    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE_			,    [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]			,    )

								_UpperCAmelCase			=    tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_ )
								self.assertListEqual(
								    SCREAMING_SNAKE_CASE_			,    [
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "<unk>",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "<unk>",
								        ".",
								    ]			,    )



							def    _a (   self )				->				Any:
								_UpperCAmelCase			=    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
										_UpperCAmelCase			=    self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_			,    **SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_			,    **SCREAMING_SNAKE_CASE_ )

										_UpperCAmelCase			=    tempfile.mkdtemp()

										_UpperCAmelCase			=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_ )

										# Checks it save with the same files + the tokenizer.json file for the fast one
										self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )
										_UpperCAmelCase			=    tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f )
										self.assertSequenceEqual(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )

										# Checks everything loads correctly in the same way
										_UpperCAmelCase			=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_ )

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ ) )

										shutil.rmtree(SCREAMING_SNAKE_CASE_ )

										# Save tokenizer rust, legacy_format=True
										_UpperCAmelCase			=    tempfile.mkdtemp()

										_UpperCAmelCase			=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_			,    legacy_format=SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_ )

										# Checks it save with the same files
										self.assertSequenceEqual(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )

										# Checks everything loads correctly in the same way
										_UpperCAmelCase			=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_ )

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ ) )

										shutil.rmtree(SCREAMING_SNAKE_CASE_ )

										# Save tokenizer rust, legacy_format=False
										_UpperCAmelCase			=    tempfile.mkdtemp()

										_UpperCAmelCase			=    tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_			,    legacy_format=SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_ )

										# Checks it saved the tokenizer.json file
										self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )

										# Checks everything loads correctly in the same way
										_UpperCAmelCase			=    tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_ )

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ ) )

										shutil.rmtree(SCREAMING_SNAKE_CASE_ )



							@require_torch
							def    _a (   self )				->				Optional[int]:
								if not self.test_seqaseq:
									return

								_UpperCAmelCase			=    self.get_tokenizers()
								for tokenizer in tokenizers:
									with self.subTest(f"{tokenizer.__class__.__name__}" ):
										# Longer text that will definitely require truncation.
										_UpperCAmelCase			=    [
										    " UN Chief Says There Is No Military Solution in Syria",
										    " Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for"
										    " Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons"
										    " will only worsen the violence and misery for millions of people.",
										]
										_UpperCAmelCase			=    [
										    "Şeful ONU declară că nu există o soluţie militară în Siria",
										    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
										    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
										    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
										]
										try:
											_UpperCAmelCase			=    tokenizer.prepare_seqaseq_batch(
											    src_texts=SCREAMING_SNAKE_CASE_			,    tgt_texts=SCREAMING_SNAKE_CASE_			,    max_length=3			,    max_target_length=10			,    return_tensors="pt"			,    src_lang="eng_Latn"			,    tgt_lang="ron_Latn"			,    )
										except NotImplementedError:
											return
										self.assertEqual(batch.input_ids.shape[1]			,    3 )
										self.assertEqual(batch.labels.shape[1]			,    10 )
										# max_target_length will default to max_length if not specified
										_UpperCAmelCase			=    tokenizer.prepare_seqaseq_batch(
										    SCREAMING_SNAKE_CASE_			,    tgt_texts=SCREAMING_SNAKE_CASE_			,    max_length=3			,    return_tensors="pt" )
										self.assertEqual(batch.input_ids.shape[1]			,    3 )
										self.assertEqual(batch.labels.shape[1]			,    3 )

										_UpperCAmelCase			=    tokenizer.prepare_seqaseq_batch(
										    src_texts=SCREAMING_SNAKE_CASE_			,    max_length=3			,    max_target_length=10			,    return_tensors="pt" )
										self.assertEqual(batch_encoder_only.input_ids.shape[1]			,    3 )
										self.assertEqual(batch_encoder_only.attention_mask.shape[1]			,    3 )
										self.assertNotIn("decoder_input_ids"			,    SCREAMING_SNAKE_CASE_ )



							@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece." )
							def    _a (   self )				->				Dict:
								pass



							def    _a (   self )				->				Optional[int]:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})" ):
										_UpperCAmelCase			=    [AddedToken("<special>"			,    lstrip=SCREAMING_SNAKE_CASE_ )]

										_UpperCAmelCase			=    self.rust_tokenizer_class.from_pretrained(
										    SCREAMING_SNAKE_CASE_			,    additional_special_tokens=SCREAMING_SNAKE_CASE_			,    **SCREAMING_SNAKE_CASE_ )
										_UpperCAmelCase			=    tokenizer_r.encode("Hey this is a <special> token" )

										_UpperCAmelCase			=    tokenizer_r.encode("<special>"			,    add_special_tokens=SCREAMING_SNAKE_CASE_ )[0]

										self.assertTrue(special_token_id in r_output )

										if self.test_slow_tokenizer:
											_UpperCAmelCase			=    self.rust_tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE_			,    additional_special_tokens=SCREAMING_SNAKE_CASE_			,    **SCREAMING_SNAKE_CASE_			,    )
											_UpperCAmelCase			=    self.tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE_			,    additional_special_tokens=SCREAMING_SNAKE_CASE_			,    **SCREAMING_SNAKE_CASE_ )

											_UpperCAmelCase			=    tokenizer_p.encode("Hey this is a <special> token" )

											_UpperCAmelCase			=    tokenizer_cr.encode("Hey this is a <special> token" )

											self.assertEqual(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )
											self.assertEqual(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )
											self.assertTrue(special_token_id in p_output )
											self.assertTrue(special_token_id in cr_output )





@require_torch
@require_sentencepiece
@require_tokenizers
class  _lowerCAmelCase    (    unittest.TestCase    ):
							lowercase_  :  int      =				'facebook/nllb-200-distilled-600M'
							lowercase_  :  List[Any]      =				[
							    ' UN Chief Says There Is No Military Solution in Syria',
							    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
							]
							lowercase_  :  Dict      =				[
							    'Şeful ONU declară că nu există o soluţie militară în Siria',
							    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
							    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
							    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
							]
							lowercase_  :  Tuple      =				[
							    256_047,
							    16_297,
							    134_408,
							    8_165,
							    248_066,
							    14_734,
							    950,
							    1_135,
							    105_721,
							    3_573,
							    83,
							    27_352,
							    108,
							    49_486,
							    2,
							]



							@classmethod
							def    _a (   cls )				->				Any:
								_UpperCAmelCase			=    NllbTokenizer.from_pretrained(
								    cls.checkpoint_name			,    src_lang="eng_Latn"			,    tgt_lang="ron_Latn" )
								_UpperCAmelCase			=    1
								return cls



							def    _a (   self )				->				Any:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]			,    256001 )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]			,    256002 )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]			,    256057 )



							def    _a (   self )				->				str:
								_UpperCAmelCase			=    self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens			,    SCREAMING_SNAKE_CASE_ )



							def    _a (   self )				->				List[str]:
								self.assertIn(SCREAMING_SNAKE_CASE_			,    self.tokenizer.all_special_ids )
								# fmt: off
								_UpperCAmelCase			=    [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
								# fmt: on

								_UpperCAmelCase			=    self.tokenizer.decode(SCREAMING_SNAKE_CASE_			,    skip_special_tokens=SCREAMING_SNAKE_CASE_ )
								_UpperCAmelCase			=    self.tokenizer.decode(generated_ids[1:]			,    skip_special_tokens=SCREAMING_SNAKE_CASE_ )
								self.assertEqual(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )
								self.assertNotIn(self.tokenizer.eos_token			,    SCREAMING_SNAKE_CASE_ )



							def    _a (   self )				->				Tuple:
								_UpperCAmelCase			=    ["this is gunna be a long sentence " * 20]
								assert isinstance(src_text[0]			,    SCREAMING_SNAKE_CASE_ )
								_UpperCAmelCase			=    10
								_UpperCAmelCase			=    self.tokenizer(SCREAMING_SNAKE_CASE_			,    max_length=SCREAMING_SNAKE_CASE_			,    truncation=SCREAMING_SNAKE_CASE_ ).input_ids[0]
								self.assertEqual(ids[-1]			,    2 )
								self.assertEqual(ids[0]			,    SCREAMING_SNAKE_CASE_ )
								self.assertEqual(len(SCREAMING_SNAKE_CASE_ )			,    SCREAMING_SNAKE_CASE_ )



							def    _a (   self )				->				List[str]:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"] )			,    [256203, 3] )



							def    _a (   self )				->				Tuple:
								_UpperCAmelCase			=    tempfile.mkdtemp()
								_UpperCAmelCase			=    self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_ )
								_UpperCAmelCase			=    NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_ )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,    SCREAMING_SNAKE_CASE_ )



							@require_torch
							def    _a (   self )				->				Optional[int]:
								_UpperCAmelCase			=    self.tokenizer(
								    self.src_text			,    text_target=self.tgt_text			,    padding=SCREAMING_SNAKE_CASE_			,    truncation=SCREAMING_SNAKE_CASE_			,    max_length=len(self.expected_src_tokens )			,    return_tensors="pt"			,    )
								_UpperCAmelCase			=    shift_tokens_right(
								    batch["labels"]			,    self.tokenizer.pad_token_id			,    self.tokenizer.lang_code_to_id["ron_Latn"] )

								self.assertIsInstance(SCREAMING_SNAKE_CASE_			,    SCREAMING_SNAKE_CASE_ )

								self.assertEqual((2, 15)			,    batch.input_ids.shape )
								self.assertEqual((2, 15)			,    batch.attention_mask.shape )
								_UpperCAmelCase			=    batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens			,    SCREAMING_SNAKE_CASE_ )
								self.assertEqual(SCREAMING_SNAKE_CASE_			,    batch.decoder_input_ids[0, 0] )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens			,    [EN_CODE] )
								self.assertEqual(self.tokenizer.suffix_tokens			,    [self.tokenizer.eos_token_id] )



							def    _a (   self )				->				Optional[int]:
								_UpperCAmelCase			=    self.tokenizer(self.src_text			,    padding=SCREAMING_SNAKE_CASE_			,    truncation=SCREAMING_SNAKE_CASE_			,    max_length=3			,    return_tensors="pt" )
								_UpperCAmelCase			=    self.tokenizer(
								    text_target=self.tgt_text			,    padding=SCREAMING_SNAKE_CASE_			,    truncation=SCREAMING_SNAKE_CASE_			,    max_length=10			,    return_tensors="pt" )
								_UpperCAmelCase			=    targets["input_ids"]
								_UpperCAmelCase			=    shift_tokens_right(
								    SCREAMING_SNAKE_CASE_			,    self.tokenizer.pad_token_id			,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]			,    )

								self.assertEqual(batch.input_ids.shape[1]			,    3 )
								self.assertEqual(batch.decoder_input_ids.shape[1]			,    10 )



							@require_torch
							def    _a (   self )				->				Optional[int]:
								_UpperCAmelCase			=    self.tokenizer._build_translation_inputs(
								    "A test"			,    return_tensors="pt"			,    src_lang="eng_Latn"			,    tgt_lang="fra_Latn" )

								self.assertEqual(
								    nested_simplify(SCREAMING_SNAKE_CASE_ )			,    {
								        # A, test, EOS, en_XX
								        "input_ids": [[256047, 70, 7356, 2]],
								        "attention_mask": [[1, 1, 1, 1]],
								        # ar_AR
								        "forced_bos_token_id": 256057,
								    }			,    )



							@require_torch
							def    _a (   self )				->				Dict:
								_UpperCAmelCase			=    True
								_UpperCAmelCase			=    self.tokenizer(
								    "UN Chief says there is no military solution in Syria"			,    src_lang="eng_Latn"			,    tgt_lang="fra_Latn" )
								self.assertEqual(
								    inputs.input_ids			,    [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047] )

								_UpperCAmelCase			=    False
								_UpperCAmelCase			=    self.tokenizer(
								    "UN Chief says there is no military solution in Syria"			,    src_lang="eng_Latn"			,    tgt_lang="fra_Latn" )
								self.assertEqual(
								    inputs.input_ids			,    [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2] )






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__a         =   logging.get_logger(__name__)

__a         =   '▁'

__a         =   {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

__a         =   {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

__a         =   {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

__a         =   {
    'ernie-m-base': 5_1_4,
    'ernie-m-large': 5_1_4,
}

__a         =   {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}
class 					lowercase__(							UpperCAmelCase				):



	"""simple docstring"""




	a     :List[str]       	=      ["input_ids"]

	a     :Dict       	=      VOCAB_FILES_NAMES
	a     :Union[str, Any]       	=      PRETRAINED_INIT_CONFIGURATION
	a     :Dict       	=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a     :List[str]       	=      PRETRAINED_VOCAB_FILES_MAP
	a     :Tuple       	=      RESOURCE_FILES_NAMES
	def __init__(			self    :     str			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]=None			,				SCREAMING_SNAKE_CASE_    :     List[Any]=False			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]="utf8"			,				SCREAMING_SNAKE_CASE_    :     List[Any]="[UNK]"			,				SCREAMING_SNAKE_CASE_    :     Dict="[SEP]"			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]="[PAD]"			,				SCREAMING_SNAKE_CASE_    :     Optional[int]="[CLS]"			,				SCREAMING_SNAKE_CASE_    :     Optional[int]="[MASK]"			,				SCREAMING_SNAKE_CASE_    :     Optional[Dict[str, Any]] = None			,				**SCREAMING_SNAKE_CASE_    :     Optional[int]			,				)			->	None:
					# Mask token behave like a normal word, i.e. include the space before it and
					# is included in the raw text, there should be a match in a non-normalized sentence.

					lowercase_	    =   {} if sp_model_kwargs is None else sp_model_kwargs
					super().__init__(
					    do_lower_case=SCREAMING_SNAKE_CASE_			,				unk_token=SCREAMING_SNAKE_CASE_			,				sep_token=SCREAMING_SNAKE_CASE_			,				pad_token=SCREAMING_SNAKE_CASE_			,				cls_token=SCREAMING_SNAKE_CASE_			,				mask_token=SCREAMING_SNAKE_CASE_			,				vocab_file=SCREAMING_SNAKE_CASE_			,				encoding=SCREAMING_SNAKE_CASE_			,				sp_model_kwargs=self.sp_model_kwargs			,				**SCREAMING_SNAKE_CASE_			,				)
					lowercase_	    =   do_lower_case
					lowercase_	    =   sentencepiece_model_ckpt
					lowercase_	    =   spm.SentencePieceProcessor(**self.sp_model_kwargs			)
					self.sp_model.Load(SCREAMING_SNAKE_CASE_			)

					# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
					if vocab_file is not None:
									lowercase_	    =   self.load_vocab(filepath=SCREAMING_SNAKE_CASE_			)
					else:
									lowercase_	    =   {self.sp_model.id_to_piece(SCREAMING_SNAKE_CASE_			): id for id in range(self.sp_model.get_piece_size()			)}
					lowercase_	    =   {v: k for k, v in self.vocab.items()}
	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     str			)			->	List[Any]:
					if text is None:
									return None

					lowercase_	    =   self.tokenize(SCREAMING_SNAKE_CASE_			)
					lowercase_    ,      lowercase_	    =   '''''', []

					for i, ch in enumerate(SCREAMING_SNAKE_CASE_			):
									if ch in self.SP_CHAR_MAPPING:
													lowercase_	    =   self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE_			)
									else:
													lowercase_	    =   unicodedata.normalize('''NFKC'''			,				SCREAMING_SNAKE_CASE_			)
									if self.is_whitespace(SCREAMING_SNAKE_CASE_			):
													continue
									normalized_text += ch
									char_mapping.extend([i] * len(SCREAMING_SNAKE_CASE_			)			)

					lowercase_    ,      lowercase_    ,      lowercase_	    =   normalized_text, [], 0

					if self.do_lower_case:
									lowercase_	    =   text.lower()

					for token in split_tokens:
									if token[:1] == "▁":
													lowercase_	    =   token[1:]
									lowercase_	    =   text[offset:].index(SCREAMING_SNAKE_CASE_			) + offset
									lowercase_	    =   start + len(SCREAMING_SNAKE_CASE_			)

									token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)			)
									lowercase_	    =   end
					return token_mapping
	@property
	def 		_lowercase    (			self    :     Union[str, Any]			)			->	Union[str, Any]:
					return len(self.vocab			)
	def 		_lowercase    (			self    :     Any			)			->	Optional[int]:
					return dict(self.vocab			,				**self.added_tokens_encoder			)
	def __getstate__(			self    :     Dict			)			->	int:
					lowercase_	    =   self.__dict__.copy()
					lowercase_	    =   None
					return state
	def __setstate__(			self    :     int			,				SCREAMING_SNAKE_CASE_    :     Any			)			->	List[Any]:
					lowercase_	    =   d

					# for backward compatibility
					if not hasattr(self			,				'''sp_model_kwargs'''			):
									lowercase_	    =   {}

					lowercase_	    =   spm.SentencePieceProcessor(**self.sp_model_kwargs			)
					self.sp_model.Load(self.sentencepiece_model_ckpt			)
	def 		_lowercase    (			self    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Tuple			)			->	str:
					return "".join((self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			) for c in text)			)
	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     List[Any]=False			,				SCREAMING_SNAKE_CASE_    :     Optional[int]=6_4			,				SCREAMING_SNAKE_CASE_    :     List[str]=0.1			)			->	Any:

					if self.sp_model_kwargs.get('''enable_sampling'''			) is True:
									lowercase_	    =   True
					if self.sp_model_kwargs.get('''alpha'''			) is not None:
									lowercase_	    =   self.sp_model_kwargs.get('''alpha'''			)
					if self.sp_model_kwargs.get('''nbest_size'''			) is not None:
									lowercase_	    =   self.sp_model_kwargs.get('''nbest_size'''			)

					if not enable_sampling:
									lowercase_	    =   self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE_			)
					else:
									lowercase_	    =   self.sp_model.SampleEncodeAsPieces(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   []
					for pi, piece in enumerate(SCREAMING_SNAKE_CASE_			):
									if piece == SPIECE_UNDERLINE:
													if not pieces[pi + 1].startswith(SCREAMING_SNAKE_CASE_			) and pi != 0:
																	new_pieces.append(SCREAMING_SNAKE_CASE_			)
																	continue
													else:
																	continue
									lowercase_	    =   0
									for i, chunk in enumerate(SCREAMING_SNAKE_CASE_			):
													if chunk == SPIECE_UNDERLINE:
																	continue
													if self.is_ch_char(SCREAMING_SNAKE_CASE_			) or self.is_punct(SCREAMING_SNAKE_CASE_			):
																	if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																					new_pieces.append(piece[lst_i:i]			)
																	new_pieces.append(SCREAMING_SNAKE_CASE_			)
																	lowercase_	    =   i + 1
													elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																	if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																					new_pieces.append(piece[lst_i:i]			)
																	lowercase_	    =   i
													elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																	if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																					new_pieces.append(piece[lst_i:i]			)
																	lowercase_	    =   i
									if len(SCREAMING_SNAKE_CASE_			) > lst_i:
													new_pieces.append(piece[lst_i:]			)
					return new_pieces
	def 		_lowercase    (			self    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     Any			)			->	Optional[Any]:
					lowercase_	    =   ''''''.join(SCREAMING_SNAKE_CASE_			).replace(SCREAMING_SNAKE_CASE_			,				''' '''			).strip()
					return out_string
	def 		_lowercase    (			self    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     Tuple			)			->	Any:
					lowercase_	    =   self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   ''''''.join(SCREAMING_SNAKE_CASE_			).replace(SCREAMING_SNAKE_CASE_			,				''' '''			).strip()
					return out_string
	def 		_lowercase    (			self    :     str			,				SCREAMING_SNAKE_CASE_    :     List[str]			)			->	List[str]:
					return self.vocab.get(SCREAMING_SNAKE_CASE_			,				self.vocab.get(self.unk_token			)			)
	def 		_lowercase    (			self    :     str			,				SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	List[Any]:
					return self.reverse_vocab.get(SCREAMING_SNAKE_CASE_			,				self.unk_token			)
	def 		_lowercase    (			self    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     Any			,				SCREAMING_SNAKE_CASE_    :     str=None			)			->	List[str]:
					if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					lowercase_	    =   [self.cls_token_id]
					lowercase_	    =   [self.sep_token_id]
					return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]=None			)			->	int:
					if offset_mapping_a is None:
									return [(0, 0)] + offset_mapping_a + [(0, 0)]

					return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
	def 		_lowercase    (			self    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     Any			,				SCREAMING_SNAKE_CASE_    :     Optional[int]=None			,				SCREAMING_SNAKE_CASE_    :     Optional[int]=False			)			->	Any:

					if already_has_special_tokens:
									if token_ids_a is not None:
													raise ValueError(
													    '''You should not supply a second sequence if the provided sequence of '''
													    '''ids is already formatted with special tokens for the model.'''			)
									return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

					if token_ids_a is not None:
									return [1] + ([0] * len(SCREAMING_SNAKE_CASE_			)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_			)) + [1]
					return [1] + ([0] * len(SCREAMING_SNAKE_CASE_			)) + [1]
	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     List[int]			,				SCREAMING_SNAKE_CASE_    :     Optional[List[int]] = None			)			->	List[int]:
					# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
					if token_ids_a is None:
									# [CLS] X [SEP]
									return (len(SCREAMING_SNAKE_CASE_			) + 2) * [0]

					# [CLS] A [SEP] [SEP] B [SEP]
					return [0] * (len(SCREAMING_SNAKE_CASE_			) + 1) + [1] * (len(SCREAMING_SNAKE_CASE_			) + 3)
	def 		_lowercase    (			self    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]			)			->	Optional[int]:
					if "\u4e00" <= char <= "\u9fff":
									return True
					return False
	def 		_lowercase    (			self    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			)			->	Tuple:
					if ("a" <= char <= "z") or ("A" <= char <= "Z"):
									return True
					return False
	def 		_lowercase    (			self    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     int			)			->	List[Any]:
					if char in ",;:.?!~，；：。？！《》【】":
									return True
					return False
	def 		_lowercase    (			self    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			)			->	Union[str, Any]:
					if char == " " or char == "\t" or char == "\n" or char == "\r":
									return True
					if len(SCREAMING_SNAKE_CASE_			) == 1:
									lowercase_	    =   unicodedata.category(SCREAMING_SNAKE_CASE_			)
									if cat == "Zs":
													return True
					return False
	def 		_lowercase    (			self    :     Any			,				SCREAMING_SNAKE_CASE_    :     Any			)			->	Tuple:
					lowercase_	    =   {}
					with io.open(SCREAMING_SNAKE_CASE_			,				'''r'''			,				encoding='''utf-8'''			) as f:
									for index, line in enumerate(SCREAMING_SNAKE_CASE_			):
													lowercase_	    =   line.rstrip('''\n'''			)
													lowercase_	    =   int(SCREAMING_SNAKE_CASE_			)

					return token_to_idx




	def 		_lowercase    (			self    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     Optional[str] = None			)			->	Tuple[str]:
					lowercase_	    =   0
					if os.path.isdir(SCREAMING_SNAKE_CASE_			):
									lowercase_	    =   os.path.join(
									    SCREAMING_SNAKE_CASE_			,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
					else:
									lowercase_	    =   (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
					with open(SCREAMING_SNAKE_CASE_			,				'''w'''			,				encoding='''utf-8'''			) as writer:
									for token, token_index in sorted(self.vocab.items()			,				key=lambda SCREAMING_SNAKE_CASE_			: kv[1]			):
													if index != token_index:
																	logger.warning(
																	    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																	    ''' Please check that the vocabulary is not corrupted!'''			)
																	lowercase_	    =   token_index
													writer.write(token + '''\n'''			)
													index += 1

					lowercase_	    =   os.path.join(SCREAMING_SNAKE_CASE_			,				'''sentencepiece.bpe.model'''			)
					with open(SCREAMING_SNAKE_CASE_			,				'''wb'''			) as fi:
									lowercase_	    =   self.sp_model.serialized_model_proto()
									fi.write(SCREAMING_SNAKE_CASE_			)

					return (vocab_file,)




