
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase_    :			str    =	logging.get_logger(__name__)

lowerCamelCase_    :			Dict    =	{"""vocab_file""": """spiece.model"""}

lowerCamelCase_    :			Any    =	{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

lowerCamelCase_    :			Optional[Any]    =	{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
lowerCamelCase_    :			Union[str, Any]    =	0
lowerCamelCase_    :			List[str]    =	1
lowerCamelCase_    :			str    =	2
lowerCamelCase_    :			Union[str, Any]    =	3
lowerCamelCase_    :			Tuple    =	4

class 							_UpperCamelCase					(       _A	):






			'''simple docstring'''

			__UpperCamelCase   :    Optional[int]          =			VOCAB_FILES_NAMES
			__UpperCamelCase   :    Union[str, Any]          =			PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase   :    str          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase   :    Tuple          =			"""left"""

			def __init__(							self			:							List[Any]    ,						snake_case_			:							Optional[int]    ,						snake_case_			:							Tuple=False    ,						snake_case_			:							List[str]=True    ,						snake_case_			:							Union[str, Any]=False    ,						snake_case_			:							Any="<s>"    ,						snake_case_			:							Any="</s>"    ,						snake_case_			:							Optional[int]="<unk>"    ,						snake_case_			:							List[Any]="<sep>"    ,						snake_case_			:							int="<pad>"    ,						snake_case_			:							Tuple="<cls>"    ,						snake_case_			:							Tuple="<mask>"    ,						snake_case_			:							Any=["<eop>", "<eod>"]    ,						snake_case_			:							Optional[Dict[str, Any]] = None    ,						**snake_case_			:							Tuple    ,						):
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCamelCase_:				int						=    AddedToken(snake_case_    ,						lstrip=snake_case_    ,						rstrip=snake_case_   ) if isinstance(snake_case_    ,						snake_case_   ) else mask_token

							UpperCamelCase_:				Union[str, Any]						=    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=snake_case_    ,						remove_space=snake_case_    ,						keep_accents=snake_case_    ,						bos_token=snake_case_    ,						eos_token=snake_case_    ,						unk_token=snake_case_    ,						sep_token=snake_case_    ,						pad_token=snake_case_    ,						cls_token=snake_case_    ,						mask_token=snake_case_    ,						additional_special_tokens=snake_case_    ,						sp_model_kwargs=self.sp_model_kwargs    ,						**snake_case_    ,						)

							UpperCamelCase_:				Dict						=    3

							UpperCamelCase_:				str						=    do_lower_case
							UpperCamelCase_:				Union[str, Any]						=    remove_space
							UpperCamelCase_:				List[Any]						=    keep_accents
							UpperCamelCase_:				Dict						=    vocab_file

							UpperCamelCase_:				Union[str, Any]						=    spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(snake_case_   )

			@property
			def    lowerCAmelCase__						(							self			:							Optional[Any]   ):
							return len(self.sp_model   )

			def    lowerCAmelCase__						(							self			:							Optional[Any]   ):
							UpperCamelCase_:				Tuple						=    {self.convert_ids_to_tokens(snake_case_   ): i for i in range(self.vocab_size   )}
							vocab.update(self.added_tokens_encoder   )
							return vocab

			def __getstate__(							self			:							Union[str, Any]   ):
							UpperCamelCase_:				List[str]						=    self.__dict__.copy()
							UpperCamelCase_:				List[str]						=    None
							return state

			def __setstate__(							self			:							Tuple    ,						snake_case_			:							List[str]   ):
							UpperCamelCase_:				Optional[Any]						=    d

							# for backward compatibility
							if not hasattr(self    ,						"""sp_model_kwargs"""   ):
											UpperCamelCase_:				Union[str, Any]						=    {}

							UpperCamelCase_:				Union[str, Any]						=    spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(self.vocab_file   )

			def    lowerCAmelCase__						(							self			:							Optional[int]    ,						snake_case_			:							List[Any]   ):
							if self.remove_space:
											UpperCamelCase_:				Dict						=    """ """.join(inputs.strip().split()   )
							else:
											UpperCamelCase_:				Optional[int]						=    inputs
							UpperCamelCase_:				Tuple						=    outputs.replace("""``"""    ,						"""\""""   ).replace("""''"""    ,						"""\""""   )

							if not self.keep_accents:
											UpperCamelCase_:				Tuple						=    unicodedata.normalize("""NFKD"""    ,						snake_case_   )
											UpperCamelCase_:				List[Any]						=    """""".join([c for c in outputs if not unicodedata.combining(snake_case_   )]   )
							if self.do_lower_case:
											UpperCamelCase_:				Optional[Any]						=    outputs.lower()

							return outputs

			def    lowerCAmelCase__						(							self			:							int    ,						snake_case_			:							str   ):
							UpperCamelCase_:				Dict						=    self.preprocess_text(snake_case_   )
							UpperCamelCase_:				List[Any]						=    self.sp_model.encode(snake_case_    ,						out_type=snake_case_   )
							UpperCamelCase_:				Optional[int]						=    []
							for piece in pieces:
											if len(snake_case_   ) > 1 and piece[-1] == str(""","""   ) and piece[-2].isdigit():
															UpperCamelCase_:				Dict						=    self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case_    ,						""""""   )   )
															if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																			if len(cur_pieces[0]   ) == 1:
																							UpperCamelCase_:				str						=    cur_pieces[1:]
																			else:
																							UpperCamelCase_:				str						=    cur_pieces[0][1:]
															cur_pieces.append(piece[-1]   )
															new_pieces.extend(snake_case_   )
											else:
															new_pieces.append(snake_case_   )

							return new_pieces

			def    lowerCAmelCase__						(							self			:							Union[str, Any]    ,						snake_case_			:							Optional[Any]   ):
							return self.sp_model.PieceToId(snake_case_   )

			def    lowerCAmelCase__						(							self			:							Union[str, Any]    ,						snake_case_			:							List[Any]   ):
							return self.sp_model.IdToPiece(snake_case_   )

			def    lowerCAmelCase__						(							self			:							int    ,						snake_case_			:							Any   ):
							UpperCamelCase_:				List[Any]						=    """""".join(snake_case_   ).replace(snake_case_    ,						""" """   ).strip()
							return out_string

			def    lowerCAmelCase__						(							self			:							Optional[Any]    ,						snake_case_			:							List[int]    ,						snake_case_			:							bool = False    ,						snake_case_			:							bool = None    ,						snake_case_			:							bool = True    ,						**snake_case_			:							List[Any]    ,						):
							UpperCamelCase_:				Dict						=    kwargs.pop("""use_source_tokenizer"""    ,						snake_case_   )

							UpperCamelCase_:				Tuple						=    self.convert_ids_to_tokens(snake_case_    ,						skip_special_tokens=snake_case_   )

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							UpperCamelCase_:				str						=    []
							UpperCamelCase_:				List[Any]						=    []
							for token in filtered_tokens:
											if skip_special_tokens and token in self.all_special_ids:
															continue
											if token in self.added_tokens_encoder:
															if current_sub_text:
																			sub_texts.append(self.convert_tokens_to_string(snake_case_   )   )
																			UpperCamelCase_:				Tuple						=    []
															sub_texts.append(snake_case_   )
											else:
															current_sub_text.append(snake_case_   )
							if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(snake_case_   )   )

							# Mimic the behavior of the Rust tokenizer:
							# By default, there are no spaces between special tokens
							UpperCamelCase_:				List[str]						=    """""".join(snake_case_   )

							UpperCamelCase_:				int						=    (
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
											UpperCamelCase_:				Any						=    self.clean_up_tokenization(snake_case_   )
											return clean_text
							else:
											return text

			def    lowerCAmelCase__						(							self			:							List[Any]    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None   ):
							UpperCamelCase_:				Optional[Any]						=    [self.sep_token_id]
							UpperCamelCase_:				List[Any]						=    [self.cls_token_id]
							if token_ids_a is None:
											return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls

			def    lowerCAmelCase__						(							self			:							Optional[int]    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None    ,						snake_case_			:							bool = False   ):

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_    ,						token_ids_a=snake_case_    ,						already_has_special_tokens=snake_case_   )

							if token_ids_a is not None:
											return ([0] * len(snake_case_   )) + [1] + ([0] * len(snake_case_   )) + [1, 1]
							return ([0] * len(snake_case_   )) + [1, 1]

			def    lowerCAmelCase__						(							self			:							List[str]    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None   ):
							UpperCamelCase_:				Dict						=    [self.sep_token_id]
							UpperCamelCase_:				Any						=    [2]

							if token_ids_a is None:
											return len(token_ids_a + sep   ) * [0] + cls_segment_id
							return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1] + cls_segment_id




			def    lowerCAmelCase__						(							self			:							Optional[Any]    ,						snake_case_			:							str    ,						snake_case_			:							Optional[str] = None   ):
							if not os.path.isdir(snake_case_   ):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
											return
							UpperCamelCase_:				int						=    os.path.join(
							    snake_case_    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

							if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case_   ) and os.path.isfile(self.vocab_file   ):
											copyfile(self.vocab_file    ,						snake_case_   )
							elif not os.path.isfile(self.vocab_file   ):
											with open(snake_case_    ,						"""wb"""   ) as fi:
															UpperCamelCase_:				List[str]						=    self.sp_model.serialized_model_proto()
															fi.write(snake_case_   )

							return (out_vocab_file,)








import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
							import torch

							from transformers.generation import DisjunctiveConstraint

@require_torch
class 							_UpperCamelCase					(       unittest.TestCase	):






			'''simple docstring'''

			def    lowerCAmelCase__						(							self			:							Dict   ):
							# For consistency across different places the DisjunctiveConstraint is called,
							# dc.token_ids is a list of integers. It is also initialized only by integers.

							UpperCamelCase_:				Optional[int]						=    [[1, 2, 4], [1, 2, 3, 4]]
							UpperCamelCase_:				List[str]						=    DisjunctiveConstraint(snake_case_   )
							self.assertTrue(isinstance(dc.token_ids    ,						snake_case_   )   )

							with self.assertRaises(snake_case_   ):
											DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]   )   )

							with self.assertRaises(snake_case_   ):
											DisjunctiveConstraint([torch.LongTensor([1, 2, 4]   ), torch.LongTensor([1, 2, 3, 4, 5]   )]   )

			def    lowerCAmelCase__						(							self			:							Optional[Any]   ):
							# We can't have constraints that are complete subsets of another. This leads to a preverse
							# interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
							# It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
							# fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
							# will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
							UpperCamelCase_:				Dict						=    [[1, 2], [1, 2, 3, 4]]

							with self.assertRaises(snake_case_   ):
											DisjunctiveConstraint(snake_case_   )  # fails here

			def    lowerCAmelCase__						(							self			:							int   ):
							UpperCamelCase_:				Dict						=    [[1, 2, 3], [1, 2, 4]]

							UpperCamelCase_:				Dict						=    DisjunctiveConstraint(snake_case_   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				str						=    dc.update(1   )
							UpperCamelCase_:				Union[str, Any]						=    stepped is True and completed is False and reset is False
							self.assertTrue(snake_case_   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.current_seq == [1]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Optional[int]						=    dc.update(2   )
							UpperCamelCase_:				Tuple						=    stepped is True and completed is False and reset is False
							self.assertTrue(snake_case_   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.current_seq == [1, 2]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Optional[int]						=    dc.update(3   )
							UpperCamelCase_:				Union[str, Any]						=    stepped is True and completed is True and reset is False
							self.assertTrue(snake_case_   )
							self.assertTrue(dc.completed   )  # Completed!
							self.assertTrue(dc.current_seq == [1, 2, 3]   )




			def    lowerCAmelCase__						(							self			:							List[str]   ):
							UpperCamelCase_:				Optional[int]						=    [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

							UpperCamelCase_:				Tuple						=    DisjunctiveConstraint(snake_case_   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Dict						=    dc.update(1   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.current_seq == [1]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Union[str, Any]						=    dc.update(2   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.current_seq == [1, 2]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Any						=    dc.update(4   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.current_seq == [1, 2, 4]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				int						=    dc.update(5   )
							self.assertTrue(dc.completed   )  # Completed!
							self.assertTrue(dc.current_seq == [1, 2, 4, 5]   )

							dc.reset()

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				int						=    dc.update(1   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.remaining() == 3   )
							self.assertTrue(dc.current_seq == [1]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				Tuple						=    dc.update(2   )
							self.assertTrue(not dc.completed   )
							self.assertTrue(dc.remaining() == 2   )
							self.assertTrue(dc.current_seq == [1, 2]   )

							UpperCamelCase_, UpperCamelCase_, UpperCamelCase_:				List[str]						=    dc.update(5   )
							self.assertTrue(dc.completed   )  # Completed!
							self.assertTrue(dc.remaining() == 0   )
							self.assertTrue(dc.current_seq == [1, 2, 5]   )






