

'''simple docstring'''

import json
import os
from typing import Optional

import numpy as np

from ...feature_extraction_utils import BatchFeature
from ...processing_utils import ProcessorMixin
from ...utils import logging
from ...utils.hub import get_file_from_repo
from ..auto import AutoTokenizer


SCREAMING_SNAKE_CASE							:				List[Any]     	=			logging.get_logger(__name__)






class    snake_case     (      _a		):






				"""simple docstring"""
				_a    	=     """AutoTokenizer"""
				_a    	=     ["""tokenizer"""]

				_a    	=     {
				    """semantic_prompt""": 1,
				    """coarse_prompt""": 2,
				    """fine_prompt""": 2,
				}



				def __init__(    self,     _lowercase,     _lowercase=None       )     ->							Dict:
											super().__init__(_lowercase       )

											SCREAMING_SNAKE_CASE_				=				speaker_embeddings



				@classmethod
				def   a__					(    cls,     _lowercase,     _lowercase="speaker_embeddings_path.json",     **_lowercase       )     ->							List[Any]:

											if speaker_embeddings_dict_path is not None:
																		SCREAMING_SNAKE_CASE_				=				get_file_from_repo(
																		    _lowercase,     _lowercase,     subfolder=kwargs.pop('subfolder',     _lowercase       ),     cache_dir=kwargs.pop('cache_dir',     _lowercase       ),     force_download=kwargs.pop('force_download',     _lowercase       ),     proxies=kwargs.pop('proxies',     _lowercase       ),     resume_download=kwargs.pop('resume_download',     _lowercase       ),     local_files_only=kwargs.pop('local_files_only',     _lowercase       ),     use_auth_token=kwargs.pop('use_auth_token',     _lowercase       ),     revision=kwargs.pop('revision',     _lowercase       ),     )
																		if speaker_embeddings_path is None:
																									logger.warning(
																									    f"""`{os.path.join(_lowercase,     _lowercase       )}` does not exists
                    , no preloaded speaker embeddings will be used - Make sure to provide a correct path to the json
                    dictionnary if wanted, otherwise set `speaker_embeddings_dict_path=None`."""       )
																									SCREAMING_SNAKE_CASE_				=				None
																		else:
																									with open(_lowercase       ) as speaker_embeddings_json:
																																SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )
											else:
																		SCREAMING_SNAKE_CASE_				=				None

											SCREAMING_SNAKE_CASE_				=				AutoTokenizer.from_pretrained(_lowercase,     **_lowercase       )

											return cls(tokenizer=_lowercase,     speaker_embeddings=_lowercase       )



				def   a__					(    self,     _lowercase,     _lowercase="speaker_embeddings_path.json",     _lowercase="speaker_embeddings",     _lowercase = False,     **_lowercase,     )     ->							str:

											if self.speaker_embeddings is not None:
																		os.makedirs(os.path.join(_lowercase,     _lowercase,     'v2'       ),     exist_ok=_lowercase       )

																		SCREAMING_SNAKE_CASE_				=				{}

																		SCREAMING_SNAKE_CASE_				=				save_directory

																		for prompt_key in self.speaker_embeddings:
																									if prompt_key != "repo_or_path":
																																SCREAMING_SNAKE_CASE_				=				self._load_voice_preset(_lowercase       )

																																SCREAMING_SNAKE_CASE_				=				{}
																																for key in self.speaker_embeddings[prompt_key]:
																																							np.save(
																																							    os.path.join(
																																							        embeddings_dict['repo_or_path'],     _lowercase,     f"""{prompt_key}_{key}"""       ),     voice_preset[key],     allow_pickle=_lowercase,     )
																																							SCREAMING_SNAKE_CASE_				=				os.path.join(_lowercase,     f"""{prompt_key}_{key}.npy"""       )

																																SCREAMING_SNAKE_CASE_				=				tmp_dict

																		with open(os.path.join(_lowercase,     _lowercase       ),     'w'       ) as fp:
																									json.dump(_lowercase,     _lowercase       )

											super().save_pretrained(_lowercase,     _lowercase,     **_lowercase       )



				def   a__					(    self,     _lowercase = None,     **_lowercase       )     ->							int:
											SCREAMING_SNAKE_CASE_				=				self.speaker_embeddings[voice_preset]

											SCREAMING_SNAKE_CASE_				=				{}
											for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
																		if key not in voice_preset_paths:
																									raise ValueError(
																									    f"""Voice preset unrecognized, missing {key} as a key in self.speaker_embeddings[{voice_preset}]."""       )

																		SCREAMING_SNAKE_CASE_				=				get_file_from_repo(
																		    self.speaker_embeddings.get('repo_or_path',     '/'       ),     voice_preset_paths[key],     subfolder=kwargs.pop('subfolder',     _lowercase       ),     cache_dir=kwargs.pop('cache_dir',     _lowercase       ),     force_download=kwargs.pop('force_download',     _lowercase       ),     proxies=kwargs.pop('proxies',     _lowercase       ),     resume_download=kwargs.pop('resume_download',     _lowercase       ),     local_files_only=kwargs.pop('local_files_only',     _lowercase       ),     use_auth_token=kwargs.pop('use_auth_token',     _lowercase       ),     revision=kwargs.pop('revision',     _lowercase       ),     )
																		if path is None:
																									raise ValueError(
																									    f"""`{os.path.join(self.speaker_embeddings.get("repo_or_path",     "/"       ),     voice_preset_paths[key]       )}` does not exists
                    , no preloaded voice preset will be used - Make sure to provide correct paths to the {voice_preset}
                    embeddings."""       )

																		SCREAMING_SNAKE_CASE_				=				np.load(_lowercase       )

											return voice_preset_dict



				def   a__					(    self,     _lowercase = None       )     ->							Any:

											for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
																		if key not in voice_preset:
																									raise ValueError(f"""Voice preset unrecognized, missing {key} as a key."""       )

																		if not isinstance(voice_preset[key],     np.ndarray       ):
																									raise ValueError(f"""{key} voice preset must be a {str(self.preset_shape[key]       )}D ndarray."""       )

																		if len(voice_preset[key].shape       ) != self.preset_shape[key]:
																									raise ValueError(f"""{key} voice preset must be a {str(self.preset_shape[key]       )}D ndarray."""       )





				def __call__(    self,     _lowercase=None,     _lowercase=None,     _lowercase="pt",     _lowercase=256,     _lowercase=False,     _lowercase=True,     _lowercase=False,     **_lowercase,     )     ->							Any:

											if voice_preset is not None and not isinstance(_lowercase,     _lowercase       ):
																		if (
																		    isinstance(_lowercase,     _lowercase       )
																		    and self.speaker_embeddings is not None
																		    and voice_preset in self.speaker_embeddings
																		):
																									SCREAMING_SNAKE_CASE_				=				self._load_voice_preset(_lowercase       )

																		else:
																									if isinstance(_lowercase,     _lowercase       ) and not voice_preset.endswith('.npz'       ):
																																SCREAMING_SNAKE_CASE_				=				voice_preset + '.npz'

																									SCREAMING_SNAKE_CASE_				=				np.load(_lowercase       )

											if voice_preset is not None:
																		self._validate_voice_preset_dict(_lowercase,     **_lowercase       )
																		SCREAMING_SNAKE_CASE_				=				BatchFeature(data=_lowercase,     tensor_type=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				self.tokenizer(
											    _lowercase,     return_tensors=_lowercase,     padding='max_length',     max_length=_lowercase,     return_attention_mask=_lowercase,     return_token_type_ids=_lowercase,     add_special_tokens=_lowercase,     **_lowercase,     )

											if voice_preset is not None:
																		SCREAMING_SNAKE_CASE_				=				voice_preset

											return encoded_text









import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A				:       List[str]      =  logging.get_logger(__name__)

A				:       List[Any]      =  {'vocab_file': 'spiece.model'}

A				:       Tuple      =  {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

A				:       Any      =  {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
A				:       Tuple      =  0
A				:       str      =  1
A				:       str      =  2
A				:       Union[str, Any]      =  3
A				:       Optional[Any]      =  4






class      UpperCamelCase(	_a ):
					snake_case_		:					Union[str, Any]    					=						VOCAB_FILES_NAMES
					snake_case_		:					List[Any]    					=						PRETRAINED_VOCAB_FILES_MAP
					snake_case_		:					str    					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					snake_case_		:					Tuple    					=						"""left"""


					def __init__(    self						:		Optional[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		Any=False  ,     SCREAMING_SNAKE_CASE						:		List[Any]=True  ,     SCREAMING_SNAKE_CASE						:		Dict=False  ,     SCREAMING_SNAKE_CASE						:		Tuple="<s>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="</s>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<unk>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<sep>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"  ,     SCREAMING_SNAKE_CASE						:		Union[str, Any]="<cls>"  ,     SCREAMING_SNAKE_CASE						:		str="<mask>"  ,     SCREAMING_SNAKE_CASE						:		int=["<eop>", "<eod>"]  ,     SCREAMING_SNAKE_CASE						:		Optional[Dict[str, Any]] = None  ,     **SCREAMING_SNAKE_CASE						:		Dict  ,     )      ->						None:




											'''simple docstring'''

											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else mask_token

											__snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=SCREAMING_SNAKE_CASE  ,     remove_space=SCREAMING_SNAKE_CASE  ,     keep_accents=SCREAMING_SNAKE_CASE  ,     bos_token=SCREAMING_SNAKE_CASE  ,     eos_token=SCREAMING_SNAKE_CASE  ,     unk_token=SCREAMING_SNAKE_CASE  ,     sep_token=SCREAMING_SNAKE_CASE  ,     pad_token=SCREAMING_SNAKE_CASE  ,     cls_token=SCREAMING_SNAKE_CASE  ,     mask_token=SCREAMING_SNAKE_CASE  ,     additional_special_tokens=SCREAMING_SNAKE_CASE  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **SCREAMING_SNAKE_CASE  ,     )

											__snake_case				=		3

											__snake_case				=		do_lower_case
											__snake_case				=		remove_space
											__snake_case				=		keep_accents
											__snake_case				=		vocab_file

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(SCREAMING_SNAKE_CASE    )


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]    )      ->						Union[str, Any]:




											'''simple docstring'''

											return len(self.sp_model    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab


					def __getstate__(    self						:		Any    )      ->						str:




											'''simple docstring'''

											__snake_case				=		self.__dict__.copy()
											__snake_case				=		None
											return state


					def __setstate__(    self						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						int:




											'''simple docstring'''

											__snake_case				=		d

											# for backward compatibility
											if not hasattr(self  ,     "sp_model_kwargs"    ):
																	__snake_case				=		{}

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						int:




											'''simple docstring'''

											if self.remove_space:
																	__snake_case				=		" ".join(inputs.strip().split()    )
											else:
																	__snake_case				=		inputs
											__snake_case				=		outputs.replace("``"  ,     "\""    ).replace("''"  ,     "\""    )

											if not self.keep_accents:
																	__snake_case				=		unicodedata.normalize("NFKD"  ,     SCREAMING_SNAKE_CASE    )
																	__snake_case				=		"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE    )]    )
											if self.do_lower_case:
																	__snake_case				=		outputs.lower()

											return outputs


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		self.preprocess_text(SCREAMING_SNAKE_CASE    )
											__snake_case				=		self.sp_model.encode(SCREAMING_SNAKE_CASE  ,     out_type=SCREAMING_SNAKE_CASE    )
											__snake_case				=		[]
											for piece in pieces:
																	if len(SCREAMING_SNAKE_CASE    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
																							__snake_case				=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE  ,     ""    )    )
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]    ) == 1:
																																			__snake_case				=		cur_pieces[1:]
																													else:
																																			__snake_case				=		cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]    )
																							new_pieces.extend(SCREAMING_SNAKE_CASE    )
																	else:
																							new_pieces.append(SCREAMING_SNAKE_CASE    )

											return new_pieces


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						Optional[Any]:




											'''simple docstring'''

											return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[Any]    )      ->						Optional[int]:




											'''simple docstring'''

											return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		int    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    ).replace(SCREAMING_SNAKE_CASE  ,     " "    ).strip()
											return out_string


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		bool = False  ,     SCREAMING_SNAKE_CASE						:		bool = None  ,     SCREAMING_SNAKE_CASE						:		bool = True  ,     **SCREAMING_SNAKE_CASE						:		Optional[int]  ,     )      ->						str:




											'''simple docstring'''

											__snake_case				=		kwargs.pop("use_source_tokenizer"  ,     SCREAMING_SNAKE_CASE    )

											__snake_case				=		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE  ,     skip_special_tokens=SCREAMING_SNAKE_CASE    )

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__snake_case				=		[]
											__snake_case				=		[]
											for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																							continue
																	if token in self.added_tokens_encoder:
																							if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )
																													__snake_case				=		[]
																							sub_texts.append(SCREAMING_SNAKE_CASE    )
																	else:
																							current_sub_text.append(SCREAMING_SNAKE_CASE    )
											if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )

											# Mimic the behavior of the Rust tokenizer:
											# By default, there are no spaces between special tokens
											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    )

											__snake_case				=		(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
																	__snake_case				=		self.clean_up_tokenization(SCREAMING_SNAKE_CASE    )
																	return clean_text
											else:
																	return text


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											if token_ids_a is None:
																	return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None  ,     SCREAMING_SNAKE_CASE						:		bool = False    )      ->						List[int]:




											'''simple docstring'''

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE  ,     token_ids_a=SCREAMING_SNAKE_CASE  ,     already_has_special_tokens=SCREAMING_SNAKE_CASE    )

											if token_ids_a is not None:
																	return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]
											return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[2]

											if token_ids_a is None:
																	return len(token_ids_a + sep    ) * [0] + cls_segment_id
											return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id






					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		Optional[str] = None    )      ->						Tuple[str]:




											'''simple docstring'''

											if not os.path.isdir(SCREAMING_SNAKE_CASE    ):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
																	return
											__snake_case				=		os.path.join(
											    SCREAMING_SNAKE_CASE  ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE    ) and os.path.isfile(self.vocab_file    ):
																	copyfile(self.vocab_file  ,     SCREAMING_SNAKE_CASE    )
											elif not os.path.isfile(self.vocab_file    ):
																	with open(SCREAMING_SNAKE_CASE  ,     "wb"    ) as fi:
																							__snake_case				=		self.sp_model.serialized_model_proto()
																							fi.write(SCREAMING_SNAKE_CASE    )

											return (out_vocab_file,)





