




import os
import re
import sys
import traceback
import warnings
from pathlib import Path
from typing import Dict, Optional, Union
from uuid import uuida

from huggingface_hub import HfFolder, ModelCard, ModelCardData, hf_hub_download, whoami
from huggingface_hub.file_download import REGEX_COMMIT_HASH
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    is_jinja_available,
)
from packaging import version
from requests import HTTPError

from .. import __version__
from .constants import (
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    SAFETENSORS_WEIGHTS_NAME,
    WEIGHTS_NAME,
)
from .import_utils import (
    ENV_VARS_TRUE_VALUES,
    _flax_version,
    _jax_version,
    _onnxruntime_version,
    _torch_version,
    is_flax_available,
    is_onnx_available,
    is_torch_available,
)
from .logging import get_logger


__SCREAMING_SNAKE_CASE 						=get_logger(__name__)


__SCREAMING_SNAKE_CASE 						=Path(__file__).parent / """model_card_template.md"""
__SCREAMING_SNAKE_CASE 						=uuida().hex
__SCREAMING_SNAKE_CASE 						=os.getenv("""HF_HUB_OFFLINE""", """""").upper() in ENV_VARS_TRUE_VALUES
__SCREAMING_SNAKE_CASE 						=os.getenv("""DISABLE_TELEMETRY""", """""").upper() in ENV_VARS_TRUE_VALUES
__SCREAMING_SNAKE_CASE 						=HUGGINGFACE_CO_RESOLVE_ENDPOINT + """/api/telemetry/"""

def 			a   (_lowerCAmelCase = None     ):
							SCREAMING_SNAKE_CASE_   	=    F"diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}"
							if DISABLE_TELEMETRY or HF_HUB_OFFLINE:
														return ua + "; telemetry/off"
							if is_torch_available():
														ua += F"; torch/{_torch_version}"
							if is_flax_available():
														ua += F"; jax/{_jax_version}"
														ua += F"; flax/{_flax_version}"
							if is_onnx_available():
														ua += F"; onnxruntime/{_onnxruntime_version}"
							# CI will set this value to True
							if os.environ.get('''DIFFUSERS_IS_CI'''  ,    ''''''     ).upper() in ENV_VARS_TRUE_VALUES:
														ua += "; is_ci/true"
							if isinstance(_lowerCAmelCase  ,    _lowerCAmelCase     ):
														ua += "; " + "; ".join(F"{k}/{v}" for k, v in user_agent.items()     )
							elif isinstance(_lowerCAmelCase  ,    _lowerCAmelCase     ):
														ua += "; " + user_agent
							return ua

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase = None  ,    _lowerCAmelCase = None     ):
							if token is None:
														SCREAMING_SNAKE_CASE_   	=    HfFolder.get_token()
							if organization is None:
														SCREAMING_SNAKE_CASE_   	=    whoami(_lowerCAmelCase     )['''name''']
														return F"{username}/{model_id}"
							else:
														return F"{organization}/{model_id}"

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase     ):
							if not is_jinja_available():
														raise ValueError(
														    '''Modelcard rendering is based on Jinja templates.'''
														    ''' Please make sure to have `jinja` installed before using `create_model_card`.'''
														    ''' To install it, please run `pip install Jinja2`.'''     )

							if hasattr(_lowerCAmelCase  ,    '''local_rank'''     ) and args.local_rank not in [-1, 0]:
														return

							SCREAMING_SNAKE_CASE_   	=    args.hub_token if hasattr(_lowerCAmelCase  ,    '''hub_token'''     ) else None
							SCREAMING_SNAKE_CASE_   	=    get_full_repo_name(_lowerCAmelCase  ,    token=_lowerCAmelCase     )

							SCREAMING_SNAKE_CASE_   	=    ModelCard.from_template(
							    card_data=ModelCardData(  # Card metadata object that will be converted to YAML block
							        language='''en'''  ,    license='''apache-2.0'''  ,    library_name='''diffusers'''  ,    tags=[]  ,    datasets=args.dataset_name  ,    metrics=[]  ,    )  ,    template_path=_lowerCAmelCase  ,    model_name=_lowerCAmelCase  ,    repo_name=_lowerCAmelCase  ,    dataset_name=args.dataset_name if hasattr(_lowerCAmelCase  ,    '''dataset_name'''     ) else None  ,    learning_rate=args.learning_rate  ,    train_batch_size=args.train_batch_size  ,    eval_batch_size=args.eval_batch_size  ,    gradient_accumulation_steps=(
							        args.gradient_accumulation_steps if hasattr(_lowerCAmelCase  ,    '''gradient_accumulation_steps'''     ) else None
							    )  ,    adam_betaa=args.adam_betaa if hasattr(_lowerCAmelCase  ,    '''adam_beta1'''     ) else None  ,    adam_betaa=args.adam_betaa if hasattr(_lowerCAmelCase  ,    '''adam_beta2'''     ) else None  ,    adam_weight_decay=args.adam_weight_decay if hasattr(_lowerCAmelCase  ,    '''adam_weight_decay'''     ) else None  ,    adam_epsilon=args.adam_epsilon if hasattr(_lowerCAmelCase  ,    '''adam_epsilon'''     ) else None  ,    lr_scheduler=args.lr_scheduler if hasattr(_lowerCAmelCase  ,    '''lr_scheduler'''     ) else None  ,    lr_warmup_steps=args.lr_warmup_steps if hasattr(_lowerCAmelCase  ,    '''lr_warmup_steps'''     ) else None  ,    ema_inv_gamma=args.ema_inv_gamma if hasattr(_lowerCAmelCase  ,    '''ema_inv_gamma'''     ) else None  ,    ema_power=args.ema_power if hasattr(_lowerCAmelCase  ,    '''ema_power'''     ) else None  ,    ema_max_decay=args.ema_max_decay if hasattr(_lowerCAmelCase  ,    '''ema_max_decay'''     ) else None  ,    mixed_precision=args.mixed_precision  ,    )

							SCREAMING_SNAKE_CASE_   	=    os.path.join(args.output_dir  ,    '''README.md'''     )
							model_card.save(_lowerCAmelCase     )

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase = None     ):
							if resolved_file is None or commit_hash is not None:
														return commit_hash
							SCREAMING_SNAKE_CASE_   	=    str(Path(_lowerCAmelCase     ).as_posix()     )
							SCREAMING_SNAKE_CASE_   	=    re.search(R'''snapshots/([^/]+)/'''  ,    _lowerCAmelCase     )
							if search is None:
														return None
							SCREAMING_SNAKE_CASE_   	=    search.groups()[0]
							return commit_hash if REGEX_COMMIT_HASH.match(_lowerCAmelCase     ) else None


# Old default cache path, potentially to be migrated.
# This logic was more or less taken from `transformers`, with the following differences:
# - Diffusers doesn't use custom environment variables to specify the cache path.
# - There is no need to migrate the cache format, just move the files to the new location.
__SCREAMING_SNAKE_CASE 						=os.path.expanduser(
    os.getenv("""HF_HOME""", os.path.join(os.getenv("""XDG_CACHE_HOME""", """~/.cache"""), """huggingface"""))
)
__SCREAMING_SNAKE_CASE 						=os.path.join(hf_cache_home, """diffusers""")

def 			a   (_lowerCAmelCase = None  ,    _lowerCAmelCase = None     ):
							if new_cache_dir is None:
														SCREAMING_SNAKE_CASE_   	=    DIFFUSERS_CACHE
							if old_cache_dir is None:
														SCREAMING_SNAKE_CASE_   	=    old_diffusers_cache

							SCREAMING_SNAKE_CASE_   	=    Path(_lowerCAmelCase     ).expanduser()
							SCREAMING_SNAKE_CASE_   	=    Path(_lowerCAmelCase     ).expanduser()
							for old_blob_path in old_cache_dir.glob('''**/blobs/*'''     ):
														if old_blob_path.is_file() and not old_blob_path.is_symlink():
																					SCREAMING_SNAKE_CASE_   	=    new_cache_dir / old_blob_path.relative_to(_lowerCAmelCase     )
																					new_blob_path.parent.mkdir(parents=_lowerCAmelCase  ,    exist_ok=_lowerCAmelCase     )
																					os.replace(_lowerCAmelCase  ,    _lowerCAmelCase     )
																					try:
																												os.symlink(_lowerCAmelCase  ,    _lowerCAmelCase     )
																					except OSError:
																												logger.warning(
																												    '''Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded.'''     )
							# At this point, old_cache_dir contains symlinks to the new cache (it can still be used).


__SCREAMING_SNAKE_CASE 						=os.path.join(DIFFUSERS_CACHE, """version_diffusers_cache.txt""")
if not os.path.isfile(cache_version_file):
			__SCREAMING_SNAKE_CASE 						=0
else:
			with open(cache_version_file) as f:
						try:
									__SCREAMING_SNAKE_CASE 						=int(f.read())
						except ValueError:
									__SCREAMING_SNAKE_CASE 						=0

if cache_version < 1:
			__SCREAMING_SNAKE_CASE 						=os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
			if old_cache_is_not_empty:
						logger.warning(
						    """The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your """
						    """existing cached models. This is a one-time operation, you can interrupt it or run it """
						    """later by calling `diffusers.utils.hub_utils.move_cache()`."""
						)
						try:
									move_cache()
						except Exception as e:
									__SCREAMING_SNAKE_CASE 						="""\n""".join(traceback.format_tb(e.__traceback__))
									logger.error(
									    f"""There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease """
									    """file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole """
									    """message and we will do our best to help."""
									)

if cache_version < 1:
			try:
						os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
						with open(cache_version_file, """w""") as f:
									f.write("""1""")
			except Exception:
						logger.warning(
						    f"""There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure """
						    """the directory exists and can be written to."""
						)

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase = None     ):
							if variant is not None:
														SCREAMING_SNAKE_CASE_   	=    weights_name.split('''.'''     )
														SCREAMING_SNAKE_CASE_   	=    splits[:-1] + [variant] + splits[-1:]
														SCREAMING_SNAKE_CASE_   	=    '''.'''.join(_lowerCAmelCase     )

							return weights_name






def 			a   (_lowerCAmelCase  ,    *,
    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase=None  ,    ):
							SCREAMING_SNAKE_CASE_   	=    str(_lowerCAmelCase     )
							if os.path.isfile(_lowerCAmelCase     ):
														return pretrained_model_name_or_path
							elif os.path.isdir(_lowerCAmelCase     ):
														if os.path.isfile(os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase     )     ):
																					# Load from a PyTorch checkpoint
																					SCREAMING_SNAKE_CASE_   	=    os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase     )
																					return model_file
														elif subfolder is not None and os.path.isfile(
														    os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     )     ):
																					SCREAMING_SNAKE_CASE_   	=    os.path.join(_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     )
																					return model_file
														else:
																					raise EnvironmentError(
																					    F"Error no file named {weights_name} found in directory {pretrained_model_name_or_path}."     )
							else:
														# 1. First check if deprecated way of loading from branches is used
														if (
														    revision in DEPRECATED_REVISION_ARGS
														    and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME)
														    and version.parse(version.parse(_lowerCAmelCase     ).base_version     ) >= version.parse('''0.20.0'''     )
														):
																					try:
																												SCREAMING_SNAKE_CASE_   	=    hf_hub_download(
																												    _lowerCAmelCase  ,    filename=_add_variant(_lowerCAmelCase  ,    _lowerCAmelCase     )  ,    cache_dir=_lowerCAmelCase  ,    force_download=_lowerCAmelCase  ,    proxies=_lowerCAmelCase  ,    resume_download=_lowerCAmelCase  ,    local_files_only=_lowerCAmelCase  ,    use_auth_token=_lowerCAmelCase  ,    user_agent=_lowerCAmelCase  ,    subfolder=_lowerCAmelCase  ,    revision=revision or commit_hash  ,    )
																												warnings.warn(
																												    F"Loading the variant {revision} from {pretrained_model_name_or_path} via `revision=\'{revision}\'` is deprecated. Loading instead from `revision=\'main\'` with `variant={revision}`. Loading model variants via `revision=\'{revision}\'` will be removed in diffusers v1. Please use `variant=\'{revision}\'` instead."  ,    _lowerCAmelCase  ,    )
																												return model_file
																					except:  # noqa: E722
																												warnings.warn(
																												    F"You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision=\'{revision}\'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant=\'{revision}\'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(_lowerCAmelCase  ,    _lowerCAmelCase     )} file in the \'main\' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title \'{pretrained_model_name_or_path} is missing {_add_variant(_lowerCAmelCase  ,    _lowerCAmelCase     )}\' so that the correct variant file can be added."  ,    _lowerCAmelCase  ,    )
														try:
																					# 2. Load model file as usual
																					SCREAMING_SNAKE_CASE_   	=    hf_hub_download(
																					    _lowerCAmelCase  ,    filename=_lowerCAmelCase  ,    cache_dir=_lowerCAmelCase  ,    force_download=_lowerCAmelCase  ,    proxies=_lowerCAmelCase  ,    resume_download=_lowerCAmelCase  ,    local_files_only=_lowerCAmelCase  ,    use_auth_token=_lowerCAmelCase  ,    user_agent=_lowerCAmelCase  ,    subfolder=_lowerCAmelCase  ,    revision=revision or commit_hash  ,    )
																					return model_file

														except RepositoryNotFoundError:
																					raise EnvironmentError(
																					    F"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier "
																					    '''listed on \'https://huggingface.co/models\'\nIf this is a private repository, make sure to pass a '''
																					    '''token having permission to this repo with `use_auth_token` or log in with `huggingface-cli '''
																					    '''login`.'''     )
														except RevisionNotFoundError:
																					raise EnvironmentError(
																					    F"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for "
																					    '''this model name. Check the model page at '''
																					    F"\'https://huggingface.co/{pretrained_model_name_or_path}\' for available revisions."     )
														except EntryNotFoundError:
																					raise EnvironmentError(
																					    F"{pretrained_model_name_or_path} does not appear to have a file named {weights_name}."     )
														except HTTPError as err:
																					raise EnvironmentError(
																					    F"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}"     )
														except ValueError:
																					raise EnvironmentError(
																					    F"We couldn\'t connect to \'{HUGGINGFACE_CO_RESOLVE_ENDPOINT}\' to load this model, couldn\'t find it"
																					    F" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a"
																					    F" directory containing a file named {weights_name} or"
																					    ''' \nCheckout your internet connection or see how to run the library in'''
																					    ''' offline mode at \'https://huggingface.co/docs/diffusers/installation#offline-mode\'.'''     )
														except EnvironmentError:
																					raise EnvironmentError(
																					    F"Can\'t load the model for \'{pretrained_model_name_or_path}\'. If you were trying to load it from "
																					    '''\'https://huggingface.co/models\', make sure you don\'t have a local directory with the same name. '''
																					    F"Otherwise, make sure \'{pretrained_model_name_or_path}\' is the correct path to a directory "
																					    F"containing a file named {weights_name}"     )











import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE 						="""▁"""

__SCREAMING_SNAKE_CASE 						={"""vocab_file""": """spiece.model"""}

__SCREAMING_SNAKE_CASE 						={
    """vocab_file""": {
        """google/reformer-crime-and-punishment""": (
            """https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"""
        )
    }
}

__SCREAMING_SNAKE_CASE 						={
    """google/reformer-crime-and-punishment""": 524_288,
}






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Any	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Optional[int]	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					["input_ids", "attention_mask"]


				def __init__(							self:   List[Any]		,					_lowerCamelCase:   List[Any]		,					_lowerCamelCase:   List[Any]="</s>"		,					_lowerCamelCase:   str="<unk>"		,					_lowerCamelCase:   Union[str, Any]=[]		,					_lowerCamelCase:   Optional[Dict[str, Any]] = None		,					**_lowerCamelCase:   List[str]		,					):
											SCREAMING_SNAKE_CASE_   	=    {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=_lowerCamelCase		,					unk_token=_lowerCamelCase		,					additional_special_tokens=_lowerCamelCase		,					sp_model_kwargs=self.sp_model_kwargs		,					**_lowerCamelCase		,					)

											SCREAMING_SNAKE_CASE_   	=    vocab_file
											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(_lowerCamelCase		)


				@property
				def     _A							(							self:   Optional[int]		):
											return self.sp_model.get_piece_size()


				def     _A							(							self:   int		):
											SCREAMING_SNAKE_CASE_   	=    {self.convert_ids_to_tokens(_lowerCamelCase		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab


				def __getstate__(							self:   Tuple		):
											SCREAMING_SNAKE_CASE_   	=    self.__dict__.copy()
											SCREAMING_SNAKE_CASE_   	=    None
											return state


				def __setstate__(							self:   Dict		,					_lowerCamelCase:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    d

											# for backward compatibility
											if not hasattr(self		,					'''sp_model_kwargs'''		):
																		SCREAMING_SNAKE_CASE_   	=    {}

											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(self.vocab_file		)


				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   str		):
											return self.sp_model.encode(_lowerCamelCase		,					out_type=_lowerCamelCase		)


				def     _A							(							self:   int		,					_lowerCamelCase:   Optional[int]		):
											return self.sp_model.piece_to_id(_lowerCamelCase		)


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   Tuple		):
											if index < self.sp_model.get_piece_size():
																		SCREAMING_SNAKE_CASE_   	=    self.sp_model.IdToPiece(_lowerCamelCase		)
											return token


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   int		):
											SCREAMING_SNAKE_CASE_   	=    []
											SCREAMING_SNAKE_CASE_   	=    ''''''
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									out_string += self.sp_model.decode(_lowerCamelCase		) + token
																									SCREAMING_SNAKE_CASE_   	=    []
																		else:
																									current_sub_tokens.append(_lowerCamelCase		)
											out_string += self.sp_model.decode(_lowerCamelCase		)
											return out_string.strip()





				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											if not os.path.isdir(_lowerCamelCase		):
																		logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
																		return
											SCREAMING_SNAKE_CASE_   	=    os.path.join(
											    _lowerCamelCase		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(_lowerCamelCase		) and os.path.isfile(self.vocab_file		):
																		copyfile(self.vocab_file		,					_lowerCamelCase		)
											elif not os.path.isfile(self.vocab_file		):
																		with open(_lowerCamelCase		,					'''wb'''		) as fi:
																									SCREAMING_SNAKE_CASE_   	=    self.sp_model.serialized_model_proto()
																									fi.write(_lowerCamelCase		)

											return (out_vocab_file,)


