






import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


__lowerCamelCase :      Union[str, Any]										=				(
    """https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py"""
)


__lowerCamelCase :      Optional[Any]										=				logging.get_logger(__name__)  # pylint: disable=invalid-name







def 					SCREAMING_SNAKE_CASE    (			):
	snake_case__			:				List[Any]       =  "https://pypi.org/pypi/diffusers/json"
	snake_case__			:				Dict       =  json.loads(request.urlopen(snake_case_       ).read()       )["releases"].keys()
	return sorted(snake_case_					,  key=lambda snake_case_       : version.Version(snake_case_       )       )







def 					SCREAMING_SNAKE_CASE    (			):
	# This function has already been executed if HF_MODULES_CACHE already is in the Python path.
	if HF_MODULES_CACHE in sys.path:
		return

	sys.path.append(snake_case_       )
	os.makedirs(snake_case_					,  exist_ok=snake_case_       )
	snake_case__			:				Optional[int]       =  Path(snake_case_       ) / "__init__.py"
	if not init_path.exists():
		init_path.touch()







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Union[str, os.PathLike]       ):
	init_hf_modules()
	snake_case__			:				Union[str, Any]       =  Path(snake_case_       ) / name
	# If the parent module does not exist yet, recursively create it.
	if not dynamic_module_path.parent.exists():
		create_dynamic_module(dynamic_module_path.parent       )
	os.makedirs(snake_case_					,  exist_ok=snake_case_       )
	snake_case__			:				str       =  dynamic_module_path / "__init__.py"
	if not init_path.exists():
		init_path.touch()







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Tuple       ):
	with open(snake_case_					,  "r"					,  encoding="utf-8"       ) as f:
		snake_case__			:				Optional[Any]       =  f.read()

	# Imports of the form `import .xxx`
	snake_case__			:				Union[str, Any]       =  re.findall("^\s*import\s+\.(\S+)\s*$"					,  snake_case_					,  flags=re.MULTILINE       )
	# Imports of the form `from .xxx import yyy`
	relative_imports += re.findall("^\s*from\s+\.(\S+)\s+import"					,  snake_case_					,  flags=re.MULTILINE       )
	# Unique-ify
	return list(set(snake_case_       )       )







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Dict       ):
	snake_case__			:				Optional[int]       =  False
	snake_case__			:				Tuple       =  [module_file]
	snake_case__			:				Tuple       =  []

	# Let's recurse through all relative imports
	while not no_change:
		snake_case__			:				Tuple       =  []
		for f in files_to_check:
			new_imports.extend(get_relative_imports(snake_case_       )       )

		snake_case__			:				int       =  Path(snake_case_       ).parent
		snake_case__			:				int       =  [str(module_path / m       ) for m in new_imports]
		snake_case__			:				Union[str, Any]       =  [f for f in new_import_files if f not in all_relative_imports]
		snake_case__			:				int       =  [F'''{f}.py''' for f in new_import_files]

		snake_case__			:				Optional[Any]       =  len(snake_case_       ) == 0
		all_relative_imports.extend(snake_case_       )

	return all_relative_imports







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Any       ):
	with open(snake_case_					,  "r"					,  encoding="utf-8"       ) as f:
		snake_case__			:				Optional[Any]       =  f.read()

	# Imports of the form `import xxx`
	snake_case__			:				Any       =  re.findall("^\s*import\s+(\S+)\s*$"					,  snake_case_					,  flags=re.MULTILINE       )
	# Imports of the form `from xxx import yyy`
	imports += re.findall("^\s*from\s+(\S+)\s+import"					,  snake_case_					,  flags=re.MULTILINE       )
	# Only keep the top-level module
	snake_case__			:				Tuple       =  [imp.split("."       )[0] for imp in imports if not imp.startswith("."       )]

	# Unique-ify and test we got them all
	snake_case__			:				Dict       =  list(set(snake_case_       )       )
	snake_case__			:				List[Any]       =  []
	for imp in imports:
		try:
			importlib.import_module(snake_case_       )
		except ImportError:
			missing_packages.append(snake_case_       )

	if len(snake_case_       ) > 0:
		raise ImportError(
		    "This modeling file requires the following packages that were not found in your environment: "
		    F'''{', '.join(snake_case_       )}. Run `pip install {' '.join(snake_case_       )}`'''       )

	return get_relative_imports(snake_case_       )







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Dict					,  snake_case_ :							List[Any]       ):
	snake_case__			:				Dict       =  module_path.replace(os.path.sep					,  "."       )
	snake_case__			:				Union[str, Any]       =  importlib.import_module(snake_case_       )

	if class_name is None:
		return find_pipeline_class(snake_case_       )
	return getattr(snake_case_					,  snake_case_       )







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							int       ):
	from ..pipelines import DiffusionPipeline

	snake_case__			:				str       =  dict(inspect.getmembers(snake_case_					,  inspect.isclass       )       )

	snake_case__			:				Tuple       =  None
	for cls_name, cls in cls_members.items():
		if (
		    cls_name != DiffusionPipeline.__name__
		    and issubclass(cls					,  snake_case_       )
		    and cls.__module__.split("."       )[0] != "diffusers"
		):
			if pipeline_class is not None:
				raise ValueError(
				    F'''Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:'''
				    F''' {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in'''
				    F''' {loaded_module}.'''       )
			snake_case__			:				Any       =  cls

	return pipeline_class







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Union[str, os.PathLike]					,  snake_case_ :							str					,  snake_case_ :							Optional[Union[str, os.PathLike]] = None					,  snake_case_ :							bool = False					,  snake_case_ :							bool = False					,  snake_case_ :							Optional[Dict[str, str]] = None					,  snake_case_ :							Optional[Union[bool, str]] = None					,  snake_case_ :							Optional[str] = None					,  snake_case_ :							bool = False					,  ):
	snake_case__			:				Optional[Any]       =  str(snake_case_       )

	snake_case__			:				Union[str, Any]       =  os.path.join(snake_case_					,  snake_case_       )

	if os.path.isfile(snake_case_       ):
		snake_case__			:				Dict       =  module_file_or_url
		snake_case__			:				Tuple       =  "local"
	elif pretrained_model_name_or_path.count("/"       ) == 0:
		snake_case__			:				Optional[Any]       =  get_diffusers_versions()
		# cut ".dev0"
		snake_case__			:				Tuple       =  "v" + ".".join(__version__.split("."       )[:3]       )

		# retrieve github version that matches
		if revision is None:
			snake_case__			:				Optional[int]       =  latest_version if latest_version[1:] in available_versions else "main"
			logger.info(F'''Defaulting to latest_version: {revision}.'''       )
		elif revision in available_versions:
			snake_case__			:				Any       =  F'''v{revision}'''
		elif revision == "main":
			snake_case__			:				Tuple       =  revision
		else:
			raise ValueError(
			    F'''`custom_revision`: {revision} does not exist. Please make sure to choose one of'''
			    F''' {', '.join(available_versions + ['main']       )}.'''       )

		# community pipeline on GitHub
		snake_case__			:				Dict       =  COMMUNITY_PIPELINES_URL.format(revision=snake_case_					,  pipeline=snake_case_       )
		try:
			snake_case__			:				Tuple       =  cached_download(
			    snake_case_					,  cache_dir=snake_case_					,  force_download=snake_case_					,  proxies=snake_case_					,  resume_download=snake_case_					,  local_files_only=snake_case_					,  use_auth_token=snake_case_					,  )
			snake_case__			:				int       =  "git"
			snake_case__			:				Optional[int]       =  pretrained_model_name_or_path + ".py"
		except EnvironmentError:
			logger.error(F'''Could not locate the {module_file} inside {pretrained_model_name_or_path}.'''       )
			raise
	else:
		try:
			# Load from URL or cache if already cached
			snake_case__			:				Dict       =  hf_hub_download(
			    snake_case_					,  snake_case_					,  cache_dir=snake_case_					,  force_download=snake_case_					,  proxies=snake_case_					,  resume_download=snake_case_					,  local_files_only=snake_case_					,  use_auth_token=snake_case_					,  )
			snake_case__			:				List[Any]       =  os.path.join("local"					,  "--".join(pretrained_model_name_or_path.split("/"       )       )       )
		except EnvironmentError:
			logger.error(F'''Could not locate the {module_file} inside {pretrained_model_name_or_path}.'''       )
			raise

    # Check we have all the requirements in our environment
	snake_case__			:				List[str]       =  check_imports(snake_case_       )

	# Now we move the module inside our cached dynamic modules.
	snake_case__			:				Any       =  DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
	create_dynamic_module(snake_case_       )
	snake_case__			:				List[Any]       =  Path(snake_case_       ) / full_submodule
	if submodule == "local" or submodule == "git":
		# We always copy local files (we could hash the file to see if there was a change, and give them the name of
		# that hash, to only copy when there is a modification but it seems overkill for now).
		# The only reason we do the copy is to avoid putting too many folders in sys.path.
		shutil.copy(snake_case_					,  submodule_path / module_file       )
		for module_needed in modules_needed:
			snake_case__			:				Dict       =  F'''{module_needed}.py'''
			shutil.copy(os.path.join(snake_case_					,  snake_case_       )					,  submodule_path / module_needed       )
	else:
		# Get the commit hash
		# TODO: we will get this info in the etag soon, so retrieve it from there and not here.
		if isinstance(snake_case_					,  snake_case_       ):
			snake_case__			:				List[Any]       =  use_auth_token
		elif use_auth_token is True:
			snake_case__			:				List[Any]       =  HfFolder.get_token()
		else:
			snake_case__			:				List[Any]       =  None

		snake_case__			:				Union[str, Any]       =  model_info(snake_case_					,  revision=snake_case_					,  token=snake_case_       ).sha

		# The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
		# benefit of versioning.
		snake_case__			:				Union[str, Any]       =  submodule_path / commit_hash
		snake_case__			:				Any       =  full_submodule + os.path.sep + commit_hash
		create_dynamic_module(snake_case_       )

		if not (submodule_path / module_file).exists():
			shutil.copy(snake_case_					,  submodule_path / module_file       )
		# Make sure we also have every file with relative
		for module_needed in modules_needed:
			if not (submodule_path / module_needed).exists():
				get_cached_module_file(
				    snake_case_					,  F'''{module_needed}.py'''					,  cache_dir=snake_case_					,  force_download=snake_case_					,  resume_download=snake_case_					,  proxies=snake_case_					,  use_auth_token=snake_case_					,  revision=snake_case_					,  local_files_only=snake_case_					,  )
	return os.path.join(snake_case_					,  snake_case_       )







def 					SCREAMING_SNAKE_CASE    (			snake_case_ :							Union[str, os.PathLike]					,  snake_case_ :							str					,  snake_case_ :							Optional[str] = None					,  snake_case_ :							Optional[Union[str, os.PathLike]] = None					,  snake_case_ :							bool = False					,  snake_case_ :							bool = False					,  snake_case_ :							Optional[Dict[str, str]] = None					,  snake_case_ :							Optional[Union[bool, str]] = None					,  snake_case_ :							Optional[str] = None					,  snake_case_ :							bool = False					,  **snake_case_ :							Any					,  ):
	snake_case__			:				Tuple       =  get_cached_module_file(
	    snake_case_					,  snake_case_					,  cache_dir=snake_case_					,  force_download=snake_case_					,  resume_download=snake_case_					,  proxies=snake_case_					,  use_auth_token=snake_case_					,  revision=snake_case_					,  local_files_only=snake_case_					,  )
	return get_class_in_module(snake_case_					,  final_module.replace(".py"					,  ""       )       )











import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__lowerCamelCase :      int										=				logging.get_logger(__name__)


__lowerCamelCase :      List[str]										=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowerCamelCase :      Any										=				{
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

__lowerCamelCase :      Optional[int]										=				{
    """allenai/led-base-16384""": 1_6384,
}



class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_						):





					"""simple docstring"""






					a_     =		VOCAB_FILES_NAMES
					a_     =		PRETRAINED_VOCAB_FILES_MAP
					a_     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					a_     =		LEDTokenizer
					a_     =		["input_ids", "attention_mask"]



					def __init__( self					:  int							,		__A					:  Optional[int]=None							,		__A					:  Any=None							,		__A					:  Any=None							,		__A					:  int="replace"							,		__A					:  Optional[Any]="<s>"							,		__A					:  List[Any]="</s>"							,		__A					:  Dict="</s>"							,		__A					:  Union[str, Any]="<s>"							,		__A					:  List[Any]="<unk>"							,		__A					:  Optional[Any]="<pad>"							,		__A					:  Optional[int]="<mask>"							,		__A					:  str=False							,		__A					:  Optional[Any]=True							,		**__A					:  List[Any]							,		):
						super().__init__(
						    __A							,		__A							,		tokenizer_file=__A							,		errors=__A							,		bos_token=__A							,		eos_token=__A							,		sep_token=__A							,		cls_token=__A							,		unk_token=__A							,		pad_token=__A							,		mask_token=__A							,		add_prefix_space=__A							,		trim_offsets=__A							,		**__A							,		)

						snake_case__			:				List[Any]       =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
						if pre_tok_state.get("add_prefix_space"							,		__A	) != add_prefix_space:
							snake_case__			:				List[str]       =  getattr(__A							,		pre_tok_state.pop("type"	)	)
							snake_case__			:				Tuple       =  add_prefix_space
							snake_case__			:				Any       =  pre_tok_class(**__A	)

						snake_case__			:				List[Any]       =  add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						snake_case__			:				str       =  "post_processor"
						snake_case__			:				Union[str, Any]       =  getattr(self.backend_tokenizer							,		__A							,		__A	)
						if tokenizer_component_instance:
							snake_case__			:				Any       =  json.loads(tokenizer_component_instance.__getstate__()	)

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
								snake_case__			:				int       =  tuple(state["sep"]	)
							if "cls" in state:
								snake_case__			:				Optional[int]       =  tuple(state["cls"]	)

							snake_case__			:				Union[str, Any]       =  False

							if state.get("add_prefix_space"							,		__A	) != add_prefix_space:
								snake_case__			:				Optional[Any]       =  add_prefix_space
								snake_case__			:				List[str]       =  True

							if state.get("trim_offsets"							,		__A	) != trim_offsets:
								snake_case__			:				str       =  trim_offsets
								snake_case__			:				Tuple       =  True

							if changes_to_apply:
								snake_case__			:				Any       =  getattr(__A							,		state.pop("type"	)	)
								snake_case__			:				Optional[Any]       =  component_class(**__A	)
								setattr(self.backend_tokenizer							,		__A							,		__A	)



					@property
					# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
					def 	_lowercase					( self					:  Union[str, Any]	):
						if self._mask_token is None:
							if self.verbose:
								logger.error("Using mask_token, but it is not set yet."	)
							return None
						return str(self._mask_token	)



					@mask_token.setter
					def 	_lowercase					( self					:  str							,		__A					:  int	):
						snake_case__			:				str       =  AddedToken(__A							,		lstrip=__A							,		rstrip=__A	) if isinstance(__A							,		__A	) else value
						snake_case__			:				Optional[int]       =  value



					def 	_lowercase					( self					:  List[str]							,		*__A					:  Union[str, Any]							,		**__A					:  int	):
						snake_case__			:				Dict       =  kwargs.get("is_split_into_words"							,		__A	)

						if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."	)

						return super()._batch_encode_plus(*__A							,		**__A	)



					def 	_lowercase					( self					:  Union[str, Any]							,		*__A					:  Tuple							,		**__A					:  Union[str, Any]	):
						snake_case__			:				List[str]       =  kwargs.get("is_split_into_words"							,		__A	)

						if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
							    "to use it with pretokenized inputs."	)

						return super()._encode_plus(*__A							,		**__A	)



					def 	_lowercase					( self					:  Union[str, Any]							,		__A					:  str							,		__A					:  Optional[str] = None	):
						snake_case__			:				str       =  self._tokenizer.model.save(__A							,		name=__A	)
						return tuple(__A	)



					def 	_lowercase					( self					:  Union[str, Any]							,		__A					:  int							,		__A					:  str=None	):
						snake_case__			:				int       =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
							return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



					def 	_lowercase					( self					:  Any							,		__A					:  List[int]							,		__A					:  Optional[List[int]] = None	):
						snake_case__			:				Optional[Any]       =  [self.sep_token_id]
						snake_case__			:				Any       =  [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep	) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]






					def 	_lowercase					( self					:  str							,		__A					:  Union[Dict[str, EncodedInput], BatchEncoding]							,		__A					:  Optional[int] = None							,		__A					:  PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,		__A					:  Optional[int] = None							,		__A					:  Optional[bool] = None							,		):
						snake_case__			:				Dict       =  super()._pad(
						    encoded_inputs=__A							,		max_length=__A							,		padding_strategy=__A							,		pad_to_multiple_of=__A							,		return_attention_mask=__A							,		)

						# Load from model defaults
						if return_attention_mask is None:
							snake_case__			:				Optional[Any]       =  "attention_mask" in self.model_input_names

						if return_attention_mask and "global_attention_mask" in encoded_inputs:
							snake_case__			:				List[Any]       =  encoded_inputs[self.model_input_names[0]]
							# `global_attention_mask` need to have the same length as other (sequential) inputs.
							snake_case__			:				Union[str, Any]       =  len(encoded_inputs["global_attention_mask"]	) != len(__A	)

							if needs_to_be_padded:
								snake_case__			:				Dict       =  len(__A	) - len(encoded_inputs["global_attention_mask"]	)

								if self.padding_side == "right":
									# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
									snake_case__			:				Optional[Any]       =  (
									    encoded_inputs["global_attention_mask"] + [-1] * difference
									)
								elif self.padding_side == "left":
									snake_case__			:				Optional[Any]       =  [-1] * difference + encoded_inputs[
									    "global_attention_mask"
									]
								else:
									raise ValueError("Invalid padding strategy:" + str(self.padding_side	)	)

						return encoded_inputs



