





'''simple docstring'''






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase       		=      logging.get_logger(__name__)

lowerCAmelCase       		=      {"""vocab_file""": """spiece.model"""}

lowerCAmelCase       		=      {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

lowerCAmelCase       		=      {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
lowerCAmelCase       		=      0
lowerCAmelCase       		=      1
lowerCAmelCase       		=      2
lowerCAmelCase       		=      3
lowerCAmelCase       		=      4
class 		lowerCamelCase  (      _A       ):
	snake_case_      =   VOCAB_FILES_NAMES
	snake_case_      =   PRETRAINED_VOCAB_FILES_MAP
	snake_case_      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case_      =   "left"



	def __init__(   self		,      a_		,      a_=False		,      a_=True		,      a_=False		,      a_="<s>"		,      a_="</s>"		,      a_="<unk>"		,      a_="<sep>"		,      a_="<pad>"		,      a_="<cls>"		,      a_="<mask>"		,      a_=["<eop>", "<eod>"]		,      a_ = None		,      **a_		,      ):
			# Mask token behave like a normal word, i.e. include the space before it
			lowerCAmelCase     :    Optional[Any]						=   AddedToken(a_		,      lstrip=a_		,      rstrip=a_	) if isinstance(a_		,      a_	) else mask_token

			lowerCAmelCase     :    Optional[int]						=   {} if sp_model_kwargs is None else sp_model_kwargs

			super().__init__(
			    do_lower_case=a_		,      remove_space=a_		,      keep_accents=a_		,      bos_token=a_		,      eos_token=a_		,      unk_token=a_		,      sep_token=a_		,      pad_token=a_		,      cls_token=a_		,      mask_token=a_		,      additional_special_tokens=a_		,      sp_model_kwargs=self.sp_model_kwargs		,      **a_		,      )

			lowerCAmelCase     :    List[Any]						=   3

			lowerCAmelCase     :    int						=   do_lower_case
			lowerCAmelCase     :    Union[str, Any]						=   remove_space
			lowerCAmelCase     :    List[str]						=   keep_accents
			lowerCAmelCase     :    Union[str, Any]						=   vocab_file

			lowerCAmelCase     :    int						=   spm.SentencePieceProcessor(**self.sp_model_kwargs	)
			self.sp_model.Load(a_	)



	@property
	def   _lowerCamelCase    (   self	):
			return len(self.sp_model	)



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    str						=   {self.convert_ids_to_tokens(a_	): i for i in range(self.vocab_size	)}
			vocab.update(self.added_tokens_encoder	)
			return vocab



	def __getstate__(   self	):
			lowerCAmelCase     :    Union[str, Any]						=   self.__dict__.copy()
			lowerCAmelCase     :    Optional[int]						=   None
			return state



	def __setstate__(   self		,      a_	):
			lowerCAmelCase     :    Tuple						=   d

			# for backward compatibility
			if not hasattr(self		,      "sp_model_kwargs"	):
					lowerCAmelCase     :    Optional[int]						=   {}

			lowerCAmelCase     :    Optional[int]						=   spm.SentencePieceProcessor(**self.sp_model_kwargs	)
			self.sp_model.Load(self.vocab_file	)



	def   _lowerCamelCase    (   self		,      a_	):
			if self.remove_space:
					lowerCAmelCase     :    int						=   " ".join(inputs.strip().split()	)
			else:
					lowerCAmelCase     :    Optional[Any]						=   inputs
			lowerCAmelCase     :    Optional[Any]						=   outputs.replace("``"		,      "\""	).replace("''"		,      "\""	)

			if not self.keep_accents:
					lowerCAmelCase     :    List[Any]						=   unicodedata.normalize("NFKD"		,      a_	)
					lowerCAmelCase     :    Union[str, Any]						=   "".join([c for c in outputs if not unicodedata.combining(a_	)]	)
			if self.do_lower_case:
					lowerCAmelCase     :    List[Any]						=   outputs.lower()

			return outputs



	def   _lowerCamelCase    (   self		,      a_	):
			lowerCAmelCase     :    Tuple						=   self.preprocess_text(a_	)
			lowerCAmelCase     :    int						=   self.sp_model.encode(a_		,      out_type=a_	)
			lowerCAmelCase     :    Optional[Any]						=   []
			for piece in pieces:
					if len(a_	) > 1 and piece[-1] == str(","	) and piece[-2].isdigit():
							lowerCAmelCase     :    int						=   self.sp_model.EncodeAsPieces(piece[:-1].replace(a_		,      ""	)	)
							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
									if len(cur_pieces[0]	) == 1:
											lowerCAmelCase     :    int						=   cur_pieces[1:]
									else:
											lowerCAmelCase     :    List[str]						=   cur_pieces[0][1:]
							cur_pieces.append(piece[-1]	)
							new_pieces.extend(a_	)
					else:
							new_pieces.append(a_	)

			return new_pieces



	def   _lowerCamelCase    (   self		,      a_	):
			return self.sp_model.PieceToId(a_	)



	def   _lowerCamelCase    (   self		,      a_	):
			return self.sp_model.IdToPiece(a_	)



	def   _lowerCamelCase    (   self		,      a_	):
			lowerCAmelCase     :    Optional[Any]						=   "".join(a_	).replace(a_		,      " "	).strip()
			return out_string



	def   _lowerCamelCase    (   self		,      a_		,      a_ = False		,      a_ = None		,      a_ = True		,      **a_		,      ):
			lowerCAmelCase     :    Any						=   kwargs.pop("use_source_tokenizer"		,      a_	)

			lowerCAmelCase     :    Tuple						=   self.convert_ids_to_tokens(a_		,      skip_special_tokens=a_	)

			# To avoid mixing byte-level and unicode for byte-level BPT
			# we need to build string separately for added tokens and byte-level tokens
			# cf. https://github.com/huggingface/transformers/issues/1133
			lowerCAmelCase     :    str						=   []
			lowerCAmelCase     :    Any						=   []
			for token in filtered_tokens:
					if skip_special_tokens and token in self.all_special_ids:
							continue
					if token in self.added_tokens_encoder:
							if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(a_	)	)
									lowerCAmelCase     :    int						=   []
							sub_texts.append(a_	)
					else:
							current_sub_text.append(a_	)
			if current_sub_text:
					sub_texts.append(self.convert_tokens_to_string(a_	)	)

			# Mimic the behavior of the Rust tokenizer:
			# By default, there are no spaces between special tokens
			lowerCAmelCase     :    Optional[Any]						=   "".join(a_	)

			lowerCAmelCase     :    Optional[Any]						=   (
			    clean_up_tokenization_spaces
			    if clean_up_tokenization_spaces is not None
			    else self.clean_up_tokenization_spaces
			)
			if clean_up_tokenization_spaces:
					lowerCAmelCase     :    Optional[int]						=   self.clean_up_tokenization(a_	)
					return clean_text
			else:
					return text



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			lowerCAmelCase     :    Optional[Any]						=   [self.sep_token_id]
			lowerCAmelCase     :    Optional[int]						=   [self.cls_token_id]
			if token_ids_a is None:
					return token_ids_a + sep + cls
			return token_ids_a + sep + token_ids_a + sep + cls



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None		,      a_ = False	):

			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=a_		,      token_ids_a=a_		,      already_has_special_tokens=a_	)

			if token_ids_a is not None:
					return ([0] * len(a_	)) + [1] + ([0] * len(a_	)) + [1, 1]
			return ([0] * len(a_	)) + [1, 1]



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			lowerCAmelCase     :    List[str]						=   [self.sep_token_id]
			lowerCAmelCase     :    Union[str, Any]						=   [2]

			if token_ids_a is None:
					return len(token_ids_a + sep	) * [0] + cls_segment_id
			return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id





	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			if not os.path.isdir(a_	):
					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
					return
			lowerCAmelCase     :    List[str]						=   os.path.join(
			    a_		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

			if os.path.abspath(self.vocab_file	) != os.path.abspath(a_	) and os.path.isfile(self.vocab_file	):
					copyfile(self.vocab_file		,      a_	)
			elif not os.path.isfile(self.vocab_file	):
					with open(a_		,      "wb"	) as fi:
							lowerCAmelCase     :    List[Any]						=   self.sp_model.serialized_model_proto()
							fi.write(a_	)

			return (out_vocab_file,)










'''simple docstring'''






from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
					if not (is_transformers_available() and is_torch_available()):
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					from ...utils.dummy_torch_and_transformers_objects import KandinskyPipeline, KandinskyPriorPipeline
else:
					from .pipeline_kandinsky import KandinskyPipeline
					from .pipeline_kandinsky_imgaimg import KandinskyImgaImgPipeline
					from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline
					from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput
					from .text_encoder import MultilingualCLIP



