



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


SCREAMING_SNAKE_CASE__      					=     logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      					=     {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__      					=     {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

SCREAMING_SNAKE_CASE__      					=     {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
SCREAMING_SNAKE_CASE__      					=     0
SCREAMING_SNAKE_CASE__      					=     1
SCREAMING_SNAKE_CASE__      					=     2
SCREAMING_SNAKE_CASE__      					=     3
SCREAMING_SNAKE_CASE__      					=     4


class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Optional[Any]						   =					VOCAB_FILES_NAMES
       lowerCAmelCase__  :List[str]						   =					PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__  :Tuple						   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__  :str						   =					"left"



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=True     ,UpperCAmelCase_=False     ,UpperCAmelCase_="<s>"     ,UpperCAmelCase_="</s>"     ,UpperCAmelCase_="<unk>"     ,UpperCAmelCase_="<sep>"     ,UpperCAmelCase_="<pad>"     ,UpperCAmelCase_="<cls>"     ,UpperCAmelCase_="<mask>"     ,UpperCAmelCase_=["<eop>", "<eod>"]     ,UpperCAmelCase_ = None     ,**UpperCAmelCase_     ,)  ->							None:
           lowercase__		       =						AddedToken(_UpperCAmelCase     ,lstrip=_UpperCAmelCase     ,rstrip=_UpperCAmelCase      ) if isinstance(_UpperCAmelCase     ,_UpperCAmelCase      ) else mask_token

           lowercase__		       =						{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=_UpperCAmelCase     ,remove_space=_UpperCAmelCase     ,keep_accents=_UpperCAmelCase     ,bos_token=_UpperCAmelCase     ,eos_token=_UpperCAmelCase     ,unk_token=_UpperCAmelCase     ,sep_token=_UpperCAmelCase     ,pad_token=_UpperCAmelCase     ,cls_token=_UpperCAmelCase     ,mask_token=_UpperCAmelCase     ,additional_special_tokens=_UpperCAmelCase     ,sp_model_kwargs=self.sp_model_kwargs     ,**_UpperCAmelCase     ,)

           lowercase__		       =						3

           lowercase__		       =						do_lower_case
           lowercase__		       =						remove_space
           lowercase__		       =						keep_accents
           lowercase__		       =						vocab_file

           lowercase__		       =						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
           self.sp_model.Load(_UpperCAmelCase      )



       @property
       def 							_a      (				self      )  ->							Optional[int]:
           return len(self.sp_model      )



       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						{self.convert_ids_to_tokens(_UpperCAmelCase      ): i for i in range(self.vocab_size      )}
           vocab.update(self.added_tokens_encoder      )
           return vocab



       def __getstate__(				self      )  ->							Union[str, Any]:
           lowercase__		       =						self.__dict__.copy()
           lowercase__		       =						None
           return state



       def __setstate__(				self     ,UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						d

           # for backward compatibility
           if not hasattr(self     ,"sp_model_kwargs"      ):
               lowercase__		       =						{}

           lowercase__		       =						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
           self.sp_model.Load(self.vocab_file      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[str]:




           if self.remove_space:
               lowercase__		       =						" ".join(inputs.strip().split()      )
           else:
               lowercase__		       =						inputs
           lowercase__		       =						outputs.replace("``"     ,"\""      ).replace("''"     ,"\""      )

           if not self.keep_accents:
               lowercase__		       =						unicodedata.normalize("NFKD"     ,_UpperCAmelCase      )
               lowercase__		       =						"".join([c for c in outputs if not unicodedata.combining(_UpperCAmelCase      )]      )
           if self.do_lower_case:
               lowercase__		       =						outputs.lower()

           return outputs



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[str]:
           lowercase__		       =						self.preprocess_text(_UpperCAmelCase      )
           lowercase__		       =						self.sp_model.encode(_UpperCAmelCase     ,out_type=_UpperCAmelCase      )
           lowercase__		       =						[]
           for piece in pieces:
               if len(_UpperCAmelCase      ) > 1 and piece[-1] == str(","      ) and piece[-2].isdigit():
                   lowercase__		       =						self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCAmelCase     ,""      )      )
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                       if len(cur_pieces[0]      ) == 1:
                           lowercase__		       =						cur_pieces[1:]
                       else:
                           lowercase__		       =						cur_pieces[0][1:]
                   cur_pieces.append(piece[-1]      )
                   new_pieces.extend(_UpperCAmelCase      )
               else:
                   new_pieces.append(_UpperCAmelCase      )

           return new_pieces



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							int:
           return self.sp_model.PieceToId(_UpperCAmelCase      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[Any]:
           return self.sp_model.IdToPiece(_UpperCAmelCase      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							Union[str, Any]:
           lowercase__		       =						"".join(_UpperCAmelCase      ).replace(_UpperCAmelCase     ," "      ).strip()
           return out_string



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = False     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = True     ,**UpperCAmelCase_     ,)  ->							str:
           lowercase__		       =						kwargs.pop("use_source_tokenizer"     ,_UpperCAmelCase      )

           lowercase__		       =						self.convert_ids_to_tokens(_UpperCAmelCase     ,skip_special_tokens=_UpperCAmelCase      )

           # To avoid mixing byte-level and unicode for byte-level BPT
           # we need to build string separately for added tokens and byte-level tokens
           # cf. https://github.com/huggingface/transformers/issues/1133
           lowercase__		       =						[]
           lowercase__		       =						[]
           for token in filtered_tokens:
               if skip_special_tokens and token in self.all_special_ids:
                   continue
               if token in self.added_tokens_encoder:
                   if current_sub_text:
                       sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase      )      )
                       lowercase__		       =						[]
                   sub_texts.append(_UpperCAmelCase      )
               else:
                   current_sub_text.append(_UpperCAmelCase      )
           if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase      )      )

           # Mimic the behavior of the Rust tokenizer:
           # By default, there are no spaces between special tokens
           lowercase__		       =						"".join(_UpperCAmelCase      )

           lowercase__		       =						(
               clean_up_tokenization_spaces
               if clean_up_tokenization_spaces is not None
               else self.clean_up_tokenization_spaces
           )
           if clean_up_tokenization_spaces:
               lowercase__		       =						self.clean_up_tokenization(_UpperCAmelCase      )
               return clean_text
           else:
               return text



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None      )  ->							List[int]:
           lowercase__		       =						[self.sep_token_id]
           lowercase__		       =						[self.cls_token_id]
           if token_ids_a is None:
               return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None     ,UpperCAmelCase_ = False      )  ->							List[int]:




           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_UpperCAmelCase     ,token_ids_a=_UpperCAmelCase     ,already_has_special_tokens=_UpperCAmelCase      )

           if token_ids_a is not None:
               return ([0] * len(_UpperCAmelCase      )) + [1] + ([0] * len(_UpperCAmelCase      )) + [1, 1]
           return ([0] * len(_UpperCAmelCase      )) + [1, 1]



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None      )  ->							List[int]:
           lowercase__		       =						[self.sep_token_id]
           lowercase__		       =						[2]

           if token_ids_a is None:
               return len(token_ids_a + sep      ) * [0] + cls_segment_id
           return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = None      )  ->							Tuple[str]:




           if not os.path.isdir(_UpperCAmelCase      ):
               logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
               return
           lowercase__		       =						os.path.join(
               _UpperCAmelCase     ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

           if os.path.abspath(self.vocab_file      ) != os.path.abspath(_UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
               copyfile(self.vocab_file     ,_UpperCAmelCase      )
           elif not os.path.isfile(self.vocab_file      ):
               with open(_UpperCAmelCase     ,"wb"      ) as fi:
                   lowercase__		       =						self.sp_model.serialized_model_proto()
                   fi.write(_UpperCAmelCase      )

           return (out_vocab_file,)









'''simple docstring'''





import copy
import random

from transformers import CLIPTokenizer


class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							List[str]:
           super().__init__(*UpperCAmelCase_     ,**UpperCAmelCase_      )
           lowercase__		       =						{}



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							str:
           lowercase__		       =						super().add_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
           if num_added_tokens == 0:
               raise ValueError(
                   F'''The tokenizer already contains the token {placeholder_token}. Please pass a different'''
                   " `placeholder_token` that is not already in the tokenizer."      )



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=1     ,**UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						[]
           if num_vec_per_token == 1:
               self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
               output.append(UpperCAmelCase_      )
           else:
               lowercase__		       =						[]
               for i in range(UpperCAmelCase_      ):
                   lowercase__		       =						placeholder_token + F'''_{i}'''
                   self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
                   output.append(UpperCAmelCase_      )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
           for token in self.token_map:
               if token in placeholder_token:
                   raise ValueError(
                       F'''The tokenizer already has placeholder token {token} that can get confused with'''
                       F''' {placeholder_token}keep placeholder tokens independent'''      )
           lowercase__		       =						output



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0      )  ->							Optional[int]:
           if isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ):
               lowercase__		       =						[]
               for i in range(len(UpperCAmelCase_      )      ):
                   output.append(self.replace_placeholder_tokens_in_text(text[i]     ,vector_shuffle=UpperCAmelCase_      )      )
               return output
           for placeholder_token in self.token_map:
               if placeholder_token in text:
                   lowercase__		       =						self.token_map[placeholder_token]
                   lowercase__		       =						tokens[: 1 + int(len(UpperCAmelCase_      ) * prop_tokens_to_load      )]
                   if vector_shuffle:
                       lowercase__		       =						copy.copy(UpperCAmelCase_      )
                       random.shuffle(UpperCAmelCase_      )
                   lowercase__		       =						text.replace(UpperCAmelCase_     ," ".join(UpperCAmelCase_      )      )
           return text



       def __call__(				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							int:
           return super().__call__(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)







       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							Optional[int]:
           return super().encode(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)




