





import argparse
import glob
import importlib.util
import os
import re

import black
from doc_builder.style_doc import style_docstrings_in_code


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
lowerCAmelCase_        =							'''src/diffusers'''
lowerCAmelCase_        =							'''.'''


# This is to make sure the diffusers module imported is the one in the repo.
lowerCAmelCase_        =							importlib.util.spec_from_file_location(
    '''diffusers''',
    os.path.join(DIFFUSERS_PATH, '''__init__.py'''),
    submodule_search_locations=[DIFFUSERS_PATH],
)
lowerCAmelCase_        =							spec.loader.load_module()






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					)						->		Any:






				"""simple docstring"""


				return line.startswith(_UpperCamelCase					) or len(_UpperCamelCase					) <= 1 or re.search(R'''^\s*\)(\s*->.*:|:)\s*$'''					,							_UpperCamelCase					) is not None






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		Optional[Any]:






				"""simple docstring"""


				snake_case_		:       Optional[Any]           =    object_name.split('''.'''					)
				snake_case_		:       Optional[int]           =    0

				# First let's find the module where our object lives.
				snake_case_		:       str           =    parts[i]
				while i < len(_UpperCamelCase					) and not os.path.isfile(os.path.join(_UpperCamelCase					,							f'''{module}.py'''					)					):
								i += 1
								if i < len(_UpperCamelCase					):
												snake_case_		:       Optional[Any]           =    os.path.join(_UpperCamelCase					,							parts[i]					)
				if i >= len(_UpperCamelCase					):
								raise ValueError(f'''`object_name` should begin with the name of a module of diffusers but got {object_name}.'''					)

				with open(os.path.join(_UpperCamelCase					,							f'''{module}.py'''					)					,							'''r'''					,							encoding='''utf-8'''					,							newline='''\n'''					) as f:
								snake_case_		:       Any           =    f.readlines()

				# Now let's find the class / func in the code!
				snake_case_		:       Optional[int]           =    ''''''
				snake_case_		:       Any           =    0
				for name in parts[i + 1 :]:
								while (
								    line_index < len(_UpperCamelCase					) and re.search(Rf'''^{indent}(class|def)\s+{name}(\(|\:)'''					,							lines[line_index]					) is None
								):
												line_index += 1
								indent += "    "
								line_index += 1

				if line_index >= len(_UpperCamelCase					):
								raise ValueError(f''' {object_name} does not match any function or class in {module}.'''					)

				# We found the beginning of the class / func, now let's find the end (when the indent diminishes).
				snake_case_		:       Optional[Any]           =    line_index
				while line_index < len(_UpperCamelCase					) and _should_continue(lines[line_index]					,							_UpperCamelCase					):
								line_index += 1
				# Clean up empty lines at the end (if any).
				while len(lines[line_index - 1]					) <= 1:
								line_index -= 1

				snake_case_		:       Any           =    lines[start_index:line_index]
				return "".join(_UpperCamelCase					)


lowerCAmelCase_        =							re.compile(r'''^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)''')
lowerCAmelCase_        =							re.compile(r'''^\s*(\S+)->(\S+)(\s+.*|$)''')
lowerCAmelCase_        =							re.compile(r'''<FILL\s+[^>]*>''')






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		List[Any]:






				"""simple docstring"""


				snake_case_		:       Tuple           =    code.split('''\n'''					)
				snake_case_		:       Optional[int]           =    0
				while idx < len(_UpperCamelCase					) and len(lines[idx]					) == 0:
								idx += 1
				if idx < len(_UpperCamelCase					):
								return re.search(R'''^(\s*)\S'''					,							lines[idx]					).groups()[0]
				return ""






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		List[str]:






				"""simple docstring"""


				snake_case_		:       List[str]           =    len(get_indent(_UpperCamelCase					)					) > 0
				if has_indent:
								snake_case_		:       Optional[Any]           =    f'''class Bla:\n{code}'''
				snake_case_		:       Union[str, Any]           =    black.Mode(target_versions={black.TargetVersion.PYaa}					,							line_length=119					,							preview=_UpperCamelCase					)
				snake_case_		:       str           =    black.format_str(_UpperCamelCase					,							mode=_UpperCamelCase					)
				snake_case_     ,				snake_case_		:       Optional[int]           =    style_docstrings_in_code(_UpperCamelCase					)
				return result[len('''class Bla:\n'''					) :] if has_indent else result






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase=False					)						->		Tuple:






				"""simple docstring"""


				with open(_UpperCamelCase					,							'''r'''					,							encoding='''utf-8'''					,							newline='''\n'''					) as f:
								snake_case_		:       int           =    f.readlines()
				snake_case_		:       Optional[Any]           =    []
				snake_case_		:       Any           =    0
				# Not a for loop cause `lines` is going to change (if `overwrite=True`).
				while line_index < len(_UpperCamelCase					):
								snake_case_		:       List[Any]           =    _re_copy_warning.search(lines[line_index]					)
								if search is None:
												line_index += 1
												continue

								# There is some copied code here, let's retrieve the original.
								snake_case_     ,				snake_case_     ,				snake_case_		:       Dict           =    search.groups()
								snake_case_		:       str           =    find_code_in_diffusers(_UpperCamelCase					)
								snake_case_		:       Tuple           =    get_indent(_UpperCamelCase					)

								snake_case_		:       List[Any]           =    line_index + 1 if indent == theoretical_indent else line_index + 2
								snake_case_		:       Optional[Any]           =    theoretical_indent
								snake_case_		:       int           =    start_index

								# Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
								snake_case_		:       Optional[int]           =    True
								while line_index < len(_UpperCamelCase					) and should_continue:
												line_index += 1
												if line_index >= len(_UpperCamelCase					):
																break
												snake_case_		:       Union[str, Any]           =    lines[line_index]
												snake_case_		:       Union[str, Any]           =    _should_continue(_UpperCamelCase					,							_UpperCamelCase					) and re.search(f'''^{indent}# End copy'''					,							_UpperCamelCase					) is None
								# Clean up empty lines at the end (if any).
								while len(lines[line_index - 1]					) <= 1:
												line_index -= 1

								snake_case_		:       Union[str, Any]           =    lines[start_index:line_index]
								snake_case_		:       List[str]           =    ''''''.join(_UpperCamelCase					)

								# Remove any nested `Copied from` comments to avoid circular copies
								snake_case_		:       List[str]           =    [line for line in theoretical_code.split('''\n'''					) if _re_copy_warning.search(_UpperCamelCase					) is None]
								snake_case_		:       str           =    '''\n'''.join(_UpperCamelCase					)

								# Before comparing, use the `replace_pattern` on the original code.
								if len(_UpperCamelCase					) > 0:
												snake_case_		:       Union[str, Any]           =    replace_pattern.replace('''with'''					,							''''''					).split(''','''					)
												snake_case_		:       List[Any]           =    [_re_replace_pattern.search(_UpperCamelCase					) for p in patterns]
												for pattern in patterns:
																if pattern is None:
																				continue
																snake_case_     ,				snake_case_     ,				snake_case_		:       Optional[int]           =    pattern.groups()
																snake_case_		:       Dict           =    re.sub(_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)
																if option.strip() == "all-casing":
																				snake_case_		:       List[Any]           =    re.sub(obja.lower()					,							obja.lower()					,							_UpperCamelCase					)
																				snake_case_		:       Optional[Any]           =    re.sub(obja.upper()					,							obja.upper()					,							_UpperCamelCase					)

            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
												snake_case_		:       str           =    blackify(lines[start_index - 1] + theoretical_code					)
												snake_case_		:       Any           =    theoretical_code[len(lines[start_index - 1]					) :]

								# Test for a diff and act accordingly.
								if observed_code != theoretical_code:
												diffs.append([object_name, start_index]					)
												if overwrite:
																snake_case_		:       int           =    lines[:start_index] + [theoretical_code] + lines[line_index:]
																snake_case_		:       Any           =    start_index + 1

				if overwrite and len(_UpperCamelCase					) > 0:
								# Warn the user a file has been modified.
								print(f'''Detected changes, rewriting {filename}.'''					)
								with open(_UpperCamelCase					,							'''w'''					,							encoding='''utf-8'''					,							newline='''\n'''					) as f:
												f.writelines(_UpperCamelCase					)
				return diffs






def 	lowerCamelCase_		(							_UpperCamelCase = False					)						->		str:






				"""simple docstring"""


				snake_case_		:       str           =    glob.glob(os.path.join(_UpperCamelCase					,							'''**/*.py'''					)					,							recursive=_UpperCamelCase					)
				snake_case_		:       Optional[Any]           =    []
				for filename in all_files:
								snake_case_		:       Optional[Any]           =    is_copy_consistent(_UpperCamelCase					,							_UpperCamelCase					)
								diffs += [f'''- {filename}: copy does not match {d[0]} at line {d[1]}''' for d in new_diffs]
				if not overwrite and len(_UpperCamelCase					) > 0:
								snake_case_		:       Optional[Any]           =    '''\n'''.join(_UpperCamelCase					)
								raise Exception(
								    '''Found the following copy inconsistencies:\n'''
								    + diff
								    + '''\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them.'''					)


if __name__ == "__main__":
	lowerCAmelCase_        =							argparse.ArgumentParser()
	parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
	lowerCAmelCase_        =							parser.parse_args()

	check_copies(args.fix_and_overwrite)



"""simple docstring"""
import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


_A							=  logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm
    import torch_xla.debug.metrics as met

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  def __init__(self							,       *_lowerCamelCase							,       _lowerCamelCase=None							,       _lowerCamelCase=None							,       _lowerCamelCase=None							,       **_lowerCamelCase	):


        """simple docstring"""

        super().__init__(*_lowerCamelCase							,       **_lowerCamelCase	)
        UpperCAmelCase__			:							Any							=    eval_examples
        UpperCAmelCase__			:							Union[str, Any]							=    post_process_function
        UpperCAmelCase__			:							int							=    quant_trainer_args
        UpperCAmelCase__			:							Dict							=    128  # default number of calibration samples



  def  _a      (self							,       _lowerCamelCase=None	):


        """simple docstring"""

        if calib_dataset is None and self.calib_dataset is None:
              raise ValueError("""Trainer: calibration requires an calib_dataset."""	)
        UpperCAmelCase__			:							List[str]							=    calib_dataset if calib_dataset is not None else self.calib_dataset

        UpperCAmelCase__			:							Optional[int]							=    self._remove_unused_columns(_lowerCamelCase							,       description="""Calibration"""	)

        return DataLoader(
            _lowerCamelCase							,       batch_size=self.args.eval_batch_size							,       collate_fn=self.data_collator							,       drop_last=self.args.dataloader_drop_last							,       num_workers=self.args.dataloader_num_workers							,       pin_memory=self.args.dataloader_pin_memory							,       shuffle=_lowerCamelCase							,       )



  def  _a      (self							,       _lowerCamelCase=None	):


        """simple docstring"""

        UpperCAmelCase__			:							int							=    self.train_dataset if calib_dataset is None else calib_dataset
        UpperCAmelCase__			:							List[str]							=    self.get_calib_dataloader(_lowerCamelCase	)

        UpperCAmelCase__			:							List[str]							=    self.model
        quant_trainer.configure_model(_lowerCamelCase							,       self.quant_trainer_args							,       calib=_lowerCamelCase	)
        model.eval()
        quant_trainer.enable_calibration(_lowerCamelCase	)

        logger.info("""***** Running calibration *****"""	)
        logger.info(F"""  Num examples = {self.calib_num}"""	)
        logger.info(F"""  Batch size = {calib_dataloader.batch_size}"""	)

        for step, inputs in enumerate(_lowerCamelCase	):
              # Prediction step
              UpperCAmelCase__					,		UpperCAmelCase__					,		UpperCAmelCase__			:							List[str]							=    self.prediction_step(_lowerCamelCase							,       _lowerCamelCase							,       prediction_loss_only=_lowerCamelCase	)
              if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                    break

        quant_trainer.finish_calibration(_lowerCamelCase							,       self.quant_trainer_args	)
        UpperCAmelCase__			:							Tuple							=    model



  def  _a      (self							,       _lowerCamelCase=None							,       _lowerCamelCase=None							,       _lowerCamelCase=None							,       _lowerCamelCase = "eval"	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    self.eval_dataset if eval_dataset is None else eval_dataset
        UpperCAmelCase__			:							List[Any]							=    self.get_eval_dataloader(_lowerCamelCase	)
        UpperCAmelCase__			:							Tuple							=    self.eval_examples if eval_examples is None else eval_examples

        # Temporarily disable metric computation, we will do it in the loop here.
        UpperCAmelCase__			:							Optional[Any]							=    self.compute_metrics
        UpperCAmelCase__			:							int							=    None
        UpperCAmelCase__			:							int							=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
              UpperCAmelCase__			:							str							=    eval_loop(
                  _lowerCamelCase							,       description="""Evaluation"""							,       prediction_loss_only=True if compute_metrics is None else None							,       ignore_keys=_lowerCamelCase							,       )
        finally:
              UpperCAmelCase__			:							List[str]							=    compute_metrics

        if self.post_process_function is not None and self.compute_metrics is not None:
              UpperCAmelCase__			:							int							=    self.post_process_function(_lowerCamelCase							,       _lowerCamelCase							,       output.predictions	)
              UpperCAmelCase__			:							List[Any]							=    self.compute_metrics(_lowerCamelCase	)

              # Prefix all keys with metric_key_prefix + '_'
              for key in list(metrics.keys()	):
                    if not key.startswith(F"""{metric_key_prefix}_"""	):
                          UpperCAmelCase__			:							Optional[int]							=    metrics.pop(_lowerCamelCase	)

              self.log(_lowerCamelCase	)
        else:
              UpperCAmelCase__			:							List[Any]							=    {}

        if self.args.tpu_metrics_debug or self.args.debug:
              # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
              xm.master_print(met.metrics_report()	)

        UpperCAmelCase__			:							Optional[int]							=    self.callback_handler.on_evaluate(self.args							,       self.state							,       self.control							,       _lowerCamelCase	)
        return metrics



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase=None							,       _lowerCamelCase = "test"	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    self.get_test_dataloader(_lowerCamelCase	)

        # Temporarily disable metric computation, we will do it in the loop here.
        UpperCAmelCase__			:							Union[str, Any]							=    self.compute_metrics
        UpperCAmelCase__			:							Optional[Any]							=    None
        UpperCAmelCase__			:							Any							=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
              UpperCAmelCase__			:							Optional[Any]							=    eval_loop(
                  _lowerCamelCase							,       description="""Prediction"""							,       prediction_loss_only=True if compute_metrics is None else None							,       ignore_keys=_lowerCamelCase							,       )
        finally:
              UpperCAmelCase__			:							Dict							=    compute_metrics

        if self.post_process_function is None or self.compute_metrics is None:
              return output

        UpperCAmelCase__			:							Dict							=    self.post_process_function(_lowerCamelCase							,       _lowerCamelCase							,       output.predictions							,       """predict"""	)
        UpperCAmelCase__			:							Any							=    self.compute_metrics(_lowerCamelCase	)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()	):
              if not key.startswith(F"""{metric_key_prefix}_"""	):
                    UpperCAmelCase__			:							Any							=    metrics.pop(_lowerCamelCase	)

        return PredictionOutput(predictions=predictions.predictions							,       label_ids=predictions.label_ids							,       metrics=_lowerCamelCase	)




  def  _a      (self							,       _lowerCamelCase="./"	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    self.eval_dataset
        UpperCAmelCase__			:							int							=    self.get_eval_dataloader(_lowerCamelCase	)

        UpperCAmelCase__			:							Union[str, Any]							=    next(iter(_lowerCamelCase	)	)

        # saving device - to make it consistent
        UpperCAmelCase__			:							int							=    torch.device("""cuda""" if torch.cuda.is_available() else """cpu"""	)

        # convert to tuple
        UpperCAmelCase__			:							List[Any]							=    tuple(v.to(_lowerCamelCase	) for k, v in batch.items()	)

        logger.info("""Converting model to be onnx compatible"""	)
        from pytorch_quantization.nn import TensorQuantizer

        UpperCAmelCase__			:							Optional[Any]							=    True

        UpperCAmelCase__			:							Tuple							=    self.model.to(_lowerCamelCase	)

        model.eval()
        model.float()

        UpperCAmelCase__			:							str							=    model.module if hasattr(_lowerCamelCase							,       """module"""	) else model
        quant_trainer.configure_model(_lowerCamelCase							,       self.quant_trainer_args	)

        UpperCAmelCase__			:							List[str]							=    os.path.join(_lowerCamelCase							,       """model.onnx"""	)
        logger.info(F"""exporting model to {output_model_file}"""	)

        UpperCAmelCase__			:							Optional[Any]							=    {0: """batch_size""", 1: """seq_len"""}

        torch.onnx.export(
            _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase							,       export_params=_lowerCamelCase							,       opset_version=13							,       do_constant_folding=_lowerCamelCase							,       input_names=["""input_ids""", """attention_mask""", """token_type_ids"""]							,       output_names=["""output_start_logits""", """output_end_logits"""]							,       dynamic_axes={
                """input_ids""": axes,
                """attention_mask""": axes,
                """token_type_ids""": axes,
                """output_start_logits""": axes,
                """output_end_logits""": axes,
            }							,       verbose=_lowerCamelCase							,       )
        logger.info("""onnx export finished"""	)

