


import argparse
import glob
import importlib.util
import os
import re

import black
from doc_builder.style_doc import style_docstrings_in_code


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
A				:       Any      =  'src/diffusers'
A				:       Dict      =  '.'


# This is to make sure the diffusers module imported is the one in the repo.
A				:       Union[str, Any]      =  importlib.util.spec_from_file_location(
    'diffusers',
    os.path.join(DIFFUSERS_PATH, '__init__.py'),
    submodule_search_locations=[DIFFUSERS_PATH],
)
A				:       Union[str, Any]      =  spec.loader.load_module()





def 						_lowerCAmelCase							(     _lowerCAmelCase   ,						_lowerCAmelCase			)    ->						Optional[int]:


						'''simple docstring'''






						return line.startswith(_lowerCAmelCase			) or len(_lowerCAmelCase			) <= 1 or re.search(r"^\s*\)(\s*->.*:|:)\s*$"   ,						_lowerCAmelCase			) is not None





def 						_lowerCAmelCase							(     _lowerCAmelCase			)    ->						List[str]:


						'''simple docstring'''






						__snake_case				=		object_name.split("."			)
						__snake_case				=		0

						# First let's find the module where our object lives.
						__snake_case				=		parts[i]
						while i < len(_lowerCAmelCase			) and not os.path.isfile(os.path.join(_lowerCAmelCase   ,						F'''{module}.py'''			)			):
												i += 1
												if i < len(_lowerCAmelCase			):
																		__snake_case				=		os.path.join(_lowerCAmelCase   ,						parts[i]			)
						if i >= len(_lowerCAmelCase			):
												raise ValueError(F'''`object_name` should begin with the name of a module of diffusers but got {object_name}.'''			)

						with open(os.path.join(_lowerCAmelCase   ,						F'''{module}.py'''			)   ,						"r"   ,						encoding="utf-8"   ,						newline="\n"			) as f:
												__snake_case				=		f.readlines()

						# Now let's find the class / func in the code!
						__snake_case				=		""
						__snake_case				=		0
						for name in parts[i + 1 :]:
												while (
												    line_index < len(_lowerCAmelCase			) and re.search(rF'''^{indent}(class|def)\s+{name}(\(|\:)'''   ,						lines[line_index]			) is None
												):
																		line_index += 1
												indent += "    "
												line_index += 1

						if line_index >= len(_lowerCAmelCase			):
												raise ValueError(F''' {object_name} does not match any function or class in {module}.'''			)

						# We found the beginning of the class / func, now let's find the end (when the indent diminishes).
						__snake_case				=		line_index
						while line_index < len(_lowerCAmelCase			) and _should_continue(lines[line_index]   ,						_lowerCAmelCase			):
												line_index += 1
						# Clean up empty lines at the end (if any).
						while len(lines[line_index - 1]			) <= 1:
												line_index -= 1

						__snake_case				=		lines[start_index:line_index]
						return "".join(_lowerCAmelCase			)


A				:       Union[str, Any]      =  re.compile(r'^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)')
A				:       Optional[Any]      =  re.compile(r'^\s*(\S+)->(\S+)(\s+.*|$)')
A				:       Tuple      =  re.compile(r'<FILL\s+[^>]*>')





def 						_lowerCAmelCase							(     _lowerCAmelCase			)    ->						str:


						'''simple docstring'''






						__snake_case				=		code.split("\n"			)
						__snake_case				=		0
						while idx < len(_lowerCAmelCase			) and len(lines[idx]			) == 0:
												idx += 1
						if idx < len(_lowerCAmelCase			):
												return re.search(r"^(\s*)\S"   ,						lines[idx]			).groups()[0]
						return ""





def 						_lowerCAmelCase							(     _lowerCAmelCase			)    ->						Union[str, Any]:


						'''simple docstring'''






						__snake_case				=		len(get_indent(_lowerCAmelCase			)			) > 0
						if has_indent:
												__snake_case				=		F'''class Bla:\n{code}'''
						__snake_case				=		black.Mode(target_versions={black.TargetVersion.PYaa}   ,						line_length=119   ,						preview=_lowerCAmelCase			)
						__snake_case				=		black.format_str(_lowerCAmelCase   ,						mode=_lowerCAmelCase			)
						__snake_case		,       __snake_case				=		style_docstrings_in_code(_lowerCAmelCase			)
						return result[len("class Bla:\n"			) :] if has_indent else result





def 						_lowerCAmelCase							(     _lowerCAmelCase   ,						_lowerCAmelCase=False			)    ->						Optional[int]:


						'''simple docstring'''






						with open(_lowerCAmelCase   ,						"r"   ,						encoding="utf-8"   ,						newline="\n"			) as f:
												__snake_case				=		f.readlines()
						__snake_case				=		[]
						__snake_case				=		0
						# Not a for loop cause `lines` is going to change (if `overwrite=True`).
						while line_index < len(_lowerCAmelCase			):
												__snake_case				=		_re_copy_warning.search(lines[line_index]			)
												if search is None:
																		line_index += 1
																		continue

												# There is some copied code here, let's retrieve the original.
												__snake_case		,       __snake_case		,       __snake_case				=		search.groups()
												__snake_case				=		find_code_in_diffusers(_lowerCAmelCase			)
												__snake_case				=		get_indent(_lowerCAmelCase			)

												__snake_case				=		line_index + 1 if indent == theoretical_indent else line_index + 2
												__snake_case				=		theoretical_indent
												__snake_case				=		start_index

												# Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
												__snake_case				=		True
												while line_index < len(_lowerCAmelCase			) and should_continue:
																		line_index += 1
																		if line_index >= len(_lowerCAmelCase			):
																								break
																		__snake_case				=		lines[line_index]
																		__snake_case				=		_should_continue(_lowerCAmelCase   ,						_lowerCAmelCase			) and re.search(F'''^{indent}# End copy'''   ,						_lowerCAmelCase			) is None
												# Clean up empty lines at the end (if any).
												while len(lines[line_index - 1]			) <= 1:
																		line_index -= 1

												__snake_case				=		lines[start_index:line_index]
												__snake_case				=		"".join(_lowerCAmelCase			)

												# Remove any nested `Copied from` comments to avoid circular copies
												__snake_case				=		[line for line in theoretical_code.split("\n"			) if _re_copy_warning.search(_lowerCAmelCase			) is None]
												__snake_case				=		"\n".join(_lowerCAmelCase			)

												# Before comparing, use the `replace_pattern` on the original code.
												if len(_lowerCAmelCase			) > 0:
																		__snake_case				=		replace_pattern.replace("with"   ,						""			).split(","			)
																		__snake_case				=		[_re_replace_pattern.search(_lowerCAmelCase			) for p in patterns]
																		for pattern in patterns:
																								if pattern is None:
																														continue
																								__snake_case		,       __snake_case		,       __snake_case				=		pattern.groups()
																								__snake_case				=		re.sub(_lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)
																								if option.strip() == "all-casing":
																														__snake_case				=		re.sub(obja.lower()   ,						obja.lower()   ,						_lowerCAmelCase			)
																														__snake_case				=		re.sub(obja.upper()   ,						obja.upper()   ,						_lowerCAmelCase			)

            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
																		__snake_case				=		blackify(lines[start_index - 1] + theoretical_code			)
																		__snake_case				=		theoretical_code[len(lines[start_index - 1]			) :]

												# Test for a diff and act accordingly.
												if observed_code != theoretical_code:
																		diffs.append([object_name, start_index]			)
																		if overwrite:
																								__snake_case				=		lines[:start_index] + [theoretical_code] + lines[line_index:]
																								__snake_case				=		start_index + 1

						if overwrite and len(_lowerCAmelCase			) > 0:
												# Warn the user a file has been modified.
												print(F'''Detected changes, rewriting {filename}.'''			)
												with open(_lowerCAmelCase   ,						"w"   ,						encoding="utf-8"   ,						newline="\n"			) as f:
																		f.writelines(_lowerCAmelCase			)
						return diffs





def 						_lowerCAmelCase							(     _lowerCAmelCase = False			)    ->						List[Any]:


						'''simple docstring'''






						__snake_case				=		glob.glob(os.path.join(_lowerCAmelCase   ,						"**/*.py"			)   ,						recursive=_lowerCAmelCase			)
						__snake_case				=		[]
						for filename in all_files:
												__snake_case				=		is_copy_consistent(_lowerCAmelCase   ,						_lowerCAmelCase			)
												diffs += [F'''- {filename}: copy does not match {d[0]} at line {d[1]}''' for d in new_diffs]
						if not overwrite and len(_lowerCAmelCase			) > 0:
												__snake_case				=		"\n".join(_lowerCAmelCase			)
												raise Exception(
												    "Found the following copy inconsistencies:\n"
												    + diff
												    + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."			)


if __name__ == "__main__":
	A				:       str      =  argparse.ArgumentParser()
	parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
	A				:       Union[str, Any]      =  parser.parse_args()

	check_copies(args.fix_and_overwrite)









import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A				:       List[str]      =  logging.get_logger(__name__)

A				:       List[Any]      =  {'vocab_file': 'spiece.model'}

A				:       Tuple      =  {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

A				:       Any      =  {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
A				:       Tuple      =  0
A				:       str      =  1
A				:       str      =  2
A				:       Union[str, Any]      =  3
A				:       Optional[Any]      =  4






class      UpperCamelCase(	_a ):
					snake_case_		:					Union[str, Any]    					=						VOCAB_FILES_NAMES
					snake_case_		:					List[Any]    					=						PRETRAINED_VOCAB_FILES_MAP
					snake_case_		:					str    					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					snake_case_		:					Tuple    					=						"""left"""


					def __init__(    self						:		Optional[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		Any=False  ,     SCREAMING_SNAKE_CASE						:		List[Any]=True  ,     SCREAMING_SNAKE_CASE						:		Dict=False  ,     SCREAMING_SNAKE_CASE						:		Tuple="<s>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="</s>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<unk>"  ,     SCREAMING_SNAKE_CASE						:		List[str]="<sep>"  ,     SCREAMING_SNAKE_CASE						:		Optional[int]="<pad>"  ,     SCREAMING_SNAKE_CASE						:		Union[str, Any]="<cls>"  ,     SCREAMING_SNAKE_CASE						:		str="<mask>"  ,     SCREAMING_SNAKE_CASE						:		int=["<eop>", "<eod>"]  ,     SCREAMING_SNAKE_CASE						:		Optional[Dict[str, Any]] = None  ,     **SCREAMING_SNAKE_CASE						:		Dict  ,     )      ->						None:




											'''simple docstring'''

											__snake_case				=		AddedToken(SCREAMING_SNAKE_CASE  ,     lstrip=SCREAMING_SNAKE_CASE  ,     rstrip=SCREAMING_SNAKE_CASE    ) if isinstance(SCREAMING_SNAKE_CASE  ,     SCREAMING_SNAKE_CASE    ) else mask_token

											__snake_case				=		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=SCREAMING_SNAKE_CASE  ,     remove_space=SCREAMING_SNAKE_CASE  ,     keep_accents=SCREAMING_SNAKE_CASE  ,     bos_token=SCREAMING_SNAKE_CASE  ,     eos_token=SCREAMING_SNAKE_CASE  ,     unk_token=SCREAMING_SNAKE_CASE  ,     sep_token=SCREAMING_SNAKE_CASE  ,     pad_token=SCREAMING_SNAKE_CASE  ,     cls_token=SCREAMING_SNAKE_CASE  ,     mask_token=SCREAMING_SNAKE_CASE  ,     additional_special_tokens=SCREAMING_SNAKE_CASE  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **SCREAMING_SNAKE_CASE  ,     )

											__snake_case				=		3

											__snake_case				=		do_lower_case
											__snake_case				=		remove_space
											__snake_case				=		keep_accents
											__snake_case				=		vocab_file

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(SCREAMING_SNAKE_CASE    )


					@property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]    )      ->						Union[str, Any]:




											'''simple docstring'''

											return len(self.sp_model    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab


					def __getstate__(    self						:		Any    )      ->						str:




											'''simple docstring'''

											__snake_case				=		self.__dict__.copy()
											__snake_case				=		None
											return state


					def __setstate__(    self						:		List[Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						int:




											'''simple docstring'''

											__snake_case				=		d

											# for backward compatibility
											if not hasattr(self  ,     "sp_model_kwargs"    ):
																	__snake_case				=		{}

											__snake_case				=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[Any]    )      ->						int:




											'''simple docstring'''

											if self.remove_space:
																	__snake_case				=		" ".join(inputs.strip().split()    )
											else:
																	__snake_case				=		inputs
											__snake_case				=		outputs.replace("``"  ,     "\""    ).replace("''"  ,     "\""    )

											if not self.keep_accents:
																	__snake_case				=		unicodedata.normalize("NFKD"  ,     SCREAMING_SNAKE_CASE    )
																	__snake_case				=		"".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE    )]    )
											if self.do_lower_case:
																	__snake_case				=		outputs.lower()

											return outputs


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		self.preprocess_text(SCREAMING_SNAKE_CASE    )
											__snake_case				=		self.sp_model.encode(SCREAMING_SNAKE_CASE  ,     out_type=SCREAMING_SNAKE_CASE    )
											__snake_case				=		[]
											for piece in pieces:
																	if len(SCREAMING_SNAKE_CASE    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
																							__snake_case				=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE  ,     ""    )    )
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]    ) == 1:
																																			__snake_case				=		cur_pieces[1:]
																													else:
																																			__snake_case				=		cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]    )
																							new_pieces.extend(SCREAMING_SNAKE_CASE    )
																	else:
																							new_pieces.append(SCREAMING_SNAKE_CASE    )

											return new_pieces


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		Optional[int]    )      ->						Optional[Any]:




											'''simple docstring'''

											return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]  ,     SCREAMING_SNAKE_CASE						:		Optional[Any]    )      ->						Optional[int]:




											'''simple docstring'''

											return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]  ,     SCREAMING_SNAKE_CASE						:		int    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    ).replace(SCREAMING_SNAKE_CASE  ,     " "    ).strip()
											return out_string


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		bool = False  ,     SCREAMING_SNAKE_CASE						:		bool = None  ,     SCREAMING_SNAKE_CASE						:		bool = True  ,     **SCREAMING_SNAKE_CASE						:		Optional[int]  ,     )      ->						str:




											'''simple docstring'''

											__snake_case				=		kwargs.pop("use_source_tokenizer"  ,     SCREAMING_SNAKE_CASE    )

											__snake_case				=		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE  ,     skip_special_tokens=SCREAMING_SNAKE_CASE    )

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__snake_case				=		[]
											__snake_case				=		[]
											for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																							continue
																	if token in self.added_tokens_encoder:
																							if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )
																													__snake_case				=		[]
																							sub_texts.append(SCREAMING_SNAKE_CASE    )
																	else:
																							current_sub_text.append(SCREAMING_SNAKE_CASE    )
											if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE    )    )

											# Mimic the behavior of the Rust tokenizer:
											# By default, there are no spaces between special tokens
											__snake_case				=		"".join(SCREAMING_SNAKE_CASE    )

											__snake_case				=		(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
																	__snake_case				=		self.clean_up_tokenization(SCREAMING_SNAKE_CASE    )
																	return clean_text
											else:
																	return text


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[self.cls_token_id]
											if token_ids_a is None:
																	return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None  ,     SCREAMING_SNAKE_CASE						:		bool = False    )      ->						List[int]:




											'''simple docstring'''

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE  ,     token_ids_a=SCREAMING_SNAKE_CASE  ,     already_has_special_tokens=SCREAMING_SNAKE_CASE    )

											if token_ids_a is not None:
																	return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]
											return ([0] * len(SCREAMING_SNAKE_CASE    )) + [1, 1]


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		List[int]  ,     SCREAMING_SNAKE_CASE						:		Optional[List[int]] = None    )      ->						List[int]:




											'''simple docstring'''

											__snake_case				=		[self.sep_token_id]
											__snake_case				=		[2]

											if token_ids_a is None:
																	return len(token_ids_a + sep    ) * [0] + cls_segment_id
											return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id






					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		Optional[str] = None    )      ->						Tuple[str]:




											'''simple docstring'''

											if not os.path.isdir(SCREAMING_SNAKE_CASE    ):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
																	return
											__snake_case				=		os.path.join(
											    SCREAMING_SNAKE_CASE  ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE    ) and os.path.isfile(self.vocab_file    ):
																	copyfile(self.vocab_file  ,     SCREAMING_SNAKE_CASE    )
											elif not os.path.isfile(self.vocab_file    ):
																	with open(SCREAMING_SNAKE_CASE  ,     "wb"    ) as fi:
																							__snake_case				=		self.sp_model.serialized_model_proto()
																							fi.write(SCREAMING_SNAKE_CASE    )

											return (out_vocab_file,)





