






"""simple docstring"""



import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


A_      =							datasets.utils.logging.get_logger(__name__)

A_      =							['''names''', '''prefix''']
A_      =							['''warn_bad_lines''', '''error_bad_lines''', '''mangle_dupe_cols''']
A_      =							['''encoding_errors''', '''on_bad_lines''']
A_      =							['''date_format''']






@dataclass
class 							__SCREAMING_SNAKE_CASE  (			datasets.BuilderConfig						):
							snake_case_       =							","
							snake_case_       =							None
							snake_case_       =							"infer"
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							True
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							False
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							True
							snake_case_       =							True
							snake_case_       =							False
							snake_case_       =							True
							snake_case_       =							None
							snake_case_       =							"."
							snake_case_       =							None
							snake_case_       =							'"'
							snake_case_       =							0
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							None
							snake_case_       =							True
							snake_case_       =							True
							snake_case_       =							0
							snake_case_       =							True
							snake_case_       =							False
							snake_case_       =							None
							snake_case_       =							10000
							snake_case_       =							None
							snake_case_       =							"strict"
							snake_case_       =							"error"
							snake_case_       =							None


							def   _UpperCamelCase    (     self :  Union[str, Any]							):
													'''simple docstring'''





													if self.delimiter is not None:
																			A__  :					Tuple											=					self.delimiter
													if self.column_names is not None:
																			A__  :					str											=					self.column_names


							@property
							def   _UpperCamelCase    (     self :  Tuple							):
													'''simple docstring'''





													A__  :					Optional[Any]											=					{
													    """sep""": self.sep,
													    """header""": self.header,
													    """names""": self.names,
													    """index_col""": self.index_col,
													    """usecols""": self.usecols,
													    """prefix""": self.prefix,
													    """mangle_dupe_cols""": self.mangle_dupe_cols,
													    """engine""": self.engine,
													    """converters""": self.converters,
													    """true_values""": self.true_values,
													    """false_values""": self.false_values,
													    """skipinitialspace""": self.skipinitialspace,
													    """skiprows""": self.skiprows,
													    """nrows""": self.nrows,
													    """na_values""": self.na_values,
													    """keep_default_na""": self.keep_default_na,
													    """na_filter""": self.na_filter,
													    """verbose""": self.verbose,
													    """skip_blank_lines""": self.skip_blank_lines,
													    """thousands""": self.thousands,
													    """decimal""": self.decimal,
													    """lineterminator""": self.lineterminator,
													    """quotechar""": self.quotechar,
													    """quoting""": self.quoting,
													    """escapechar""": self.escapechar,
													    """comment""": self.comment,
													    """encoding""": self.encoding,
													    """dialect""": self.dialect,
													    """error_bad_lines""": self.error_bad_lines,
													    """warn_bad_lines""": self.warn_bad_lines,
													    """skipfooter""": self.skipfooter,
													    """doublequote""": self.doublequote,
													    """memory_map""": self.memory_map,
													    """float_precision""": self.float_precision,
													    """chunksize""": self.chunksize,
													    """encoding_errors""": self.encoding_errors,
													    """on_bad_lines""": self.on_bad_lines,
													    """date_format""": self.date_format,
													}

													# some kwargs must not be passed if they don't have a default value
													# some others are deprecated and we can also not pass them if they are the default value
													for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
																			if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()     ,      snake_case							):
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
													if not (datasets.config.PANDAS_VERSION.major >= 2):
																			for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
													if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
																			for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																									del pd_read_csv_kwargs[pd_read_csv_parameter]

													return pd_read_csv_kwargs






class 							__SCREAMING_SNAKE_CASE  (			datasets.ArrowBasedBuilder						):
							snake_case_       =							CsvConfig


							def   _UpperCamelCase    (     self :  Any							):
													'''simple docstring'''





													return datasets.DatasetInfo(features=self.config.features							)


							def   _UpperCamelCase    (     self :  Optional[Any]     ,      snake_case :  Dict							):
													'''simple docstring'''





													if not self.config.data_files:
																			raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'							)
													A__  :					int											=					dl_manager.download_and_extract(self.config.data_files							)
													if isinstance(snake_case     ,      (str, list, tuple)							):
																			A__  :					Optional[Any]											=					data_files
																			if isinstance(snake_case     ,      snake_case							):
																									A__  :					List[str]											=					[files]
																			A__  :					Union[str, Any]											=					[dl_manager.iter_files(snake_case							) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,      gen_kwargs={"""files""": files}							)]
													A__  :					str											=					[]
													for split_name, files in data_files.items():
																			if isinstance(snake_case     ,      snake_case							):
																									A__  :					List[Any]											=					[files]
																			A__  :					List[str]											=					[dl_manager.iter_files(snake_case							) for file in files]
																			splits.append(datasets.SplitGenerator(name=snake_case     ,      gen_kwargs={"""files""": files}							)							)
													return splits


							def   _UpperCamelCase    (     self :  List[str]     ,      snake_case :  pa.Table							):
													'''simple docstring'''





													if self.config.features is not None:
																			A__  :					int											=					self.config.features.arrow_schema
																			if all(not require_storage_cast(snake_case							) for feature in self.config.features.values()							):
																									# cheaper cast
																									A__  :					int											=					pa.Table.from_arrays([pa_table[field.name] for field in schema]     ,      schema=snake_case							)
																			else:
																									# more expensive cast; allows str <-> int/float or str to Audio for example
																									A__  :					Dict											=					table_cast(snake_case     ,      snake_case							)
													return pa_table


							def   _UpperCamelCase    (     self :  int     ,      snake_case :  Dict							):
													'''simple docstring'''





													A__  :					Optional[int]											=					self.config.features.arrow_schema if self.config.features else None
													# dtype allows reading an int column as str
													A__  :					int											=					(
													    {
													        name: dtype.to_pandas_dtype() if not require_storage_cast(snake_case							) else object
													        for name, dtype, feature in zip(schema.names     ,      schema.types     ,      self.config.features.values()							)
													    }
													    if schema is not None
													    else None
													)
													for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case							)							):
																			A__  :					Optional[int]											=					pd.read_csv(snake_case     ,      iterator=snake_case     ,      dtype=snake_case     ,      **self.config.pd_read_csv_kwargs							)
																			try:
																									for batch_idx, df in enumerate(snake_case							):
																															A__  :					Union[str, Any]											=					pa.Table.from_pandas(snake_case							)
																															# Uncomment for debugging (will print the Arrow table size and elements)
																															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(snake_case							)
																			except ValueError as e:
																									logger.error(F'Failed to read file \'{file}\' with error {type(snake_case							)}: {e}'							)
																									raise














"""simple docstring"""



import argparse
import glob
import importlib.util
import os
import re

import black
from doc_builder.style_doc import style_docstrings_in_code


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
A_      =							'''src/diffusers'''
A_      =							'''.'''


# This is to make sure the diffusers module imported is the one in the repo.
A_      =							importlib.util.spec_from_file_location(
    '''diffusers''',
    os.path.join(DIFFUSERS_PATH, '''__init__.py'''),
    submodule_search_locations=[DIFFUSERS_PATH],
)
A_      =							spec.loader.load_module()




def        _lowerCAmelCase						(							UpperCAmelCase__    :   Optional[int],  UpperCAmelCase__    :   Optional[Any] )						->Any:
						return line.startswith(UpperCAmelCase__ ) or len(UpperCAmelCase__ ) <= 1 or re.search(R"""^\s*\)(\s*->.*:|:)\s*$""",  UpperCAmelCase__ ) is not None




def        _lowerCAmelCase						(							UpperCAmelCase__    :   List[str] )						->Union[str, Any]:
						A__  :					Any											=					object_name.split(""".""" )
						A__  :					int											=					0

						# First let's find the module where our object lives.
						A__  :					str											=					parts[i]
						while i < len(UpperCAmelCase__ ) and not os.path.isfile(os.path.join(UpperCAmelCase__,  f'{module}.py' ) ):
												i += 1
												if i < len(UpperCAmelCase__ ):
																		A__  :					Union[str, Any]											=					os.path.join(UpperCAmelCase__,  parts[i] )
						if i >= len(UpperCAmelCase__ ):
												raise ValueError(f'`object_name` should begin with the name of a module of diffusers but got {object_name}.' )

						with open(os.path.join(UpperCAmelCase__,  f'{module}.py' ),  """r""",  encoding="""utf-8""",  newline="""\n""" ) as f:
												A__  :					List[Any]											=					f.readlines()

						# Now let's find the class / func in the code!
						A__  :					Optional[Any]											=					""""""
						A__  :					Any											=					0
						for name in parts[i + 1 :]:
												while (
												    line_index < len(UpperCAmelCase__ ) and re.search(Rf'^{indent}(class|def)\s+{name}(\(|\:)',  lines[line_index] ) is None
												):
																		line_index += 1
												indent += "    "
												line_index += 1

						if line_index >= len(UpperCAmelCase__ ):
												raise ValueError(f' {object_name} does not match any function or class in {module}.' )

						# We found the beginning of the class / func, now let's find the end (when the indent diminishes).
						A__  :					List[Any]											=					line_index
						while line_index < len(UpperCAmelCase__ ) and _should_continue(lines[line_index],  UpperCAmelCase__ ):
												line_index += 1
						# Clean up empty lines at the end (if any).
						while len(lines[line_index - 1] ) <= 1:
												line_index -= 1

						A__  :					List[Any]											=					lines[start_index:line_index]
						return "".join(UpperCAmelCase__ )


A_      =							re.compile(r'''^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)''')
A_      =							re.compile(r'''^\s*(\S+)->(\S+)(\s+.*|$)''')
A_      =							re.compile(r'''<FILL\s+[^>]*>''')




def        _lowerCAmelCase						(							UpperCAmelCase__    :   List[str] )						->Optional[Any]:
						A__  :					Dict											=					code.split("""\n""" )
						A__  :					List[Any]											=					0
						while idx < len(UpperCAmelCase__ ) and len(lines[idx] ) == 0:
												idx += 1
						if idx < len(UpperCAmelCase__ ):
												return re.search(R"""^(\s*)\S""",  lines[idx] ).groups()[0]
						return ""




def        _lowerCAmelCase						(							UpperCAmelCase__    :   Optional[Any] )						->int:
						A__  :					str											=					len(get_indent(UpperCAmelCase__ ) ) > 0
						if has_indent:
												A__  :					Union[str, Any]											=					f'class Bla:\n{code}'
						A__  :					Optional[Any]											=					black.Mode(target_versions={black.TargetVersion.PYaa},  line_length=1_1_9,  preview=UpperCAmelCase__ )
						A__  :					Tuple											=					black.format_str(UpperCAmelCase__,  mode=UpperCAmelCase__ )
						A__					,					A__  :					List[Any]											=					style_docstrings_in_code(UpperCAmelCase__ )
						return result[len("""class Bla:\n""" ) :] if has_indent else result




def        _lowerCAmelCase						(							UpperCAmelCase__    :   Any,  UpperCAmelCase__    :   Dict=False )						->List[Any]:
						with open(UpperCAmelCase__,  """r""",  encoding="""utf-8""",  newline="""\n""" ) as f:
												A__  :					int											=					f.readlines()
						A__  :					Dict											=					[]
						A__  :					List[str]											=					0
						# Not a for loop cause `lines` is going to change (if `overwrite=True`).
						while line_index < len(UpperCAmelCase__ ):
												A__  :					Dict											=					_re_copy_warning.search(lines[line_index] )
												if search is None:
																		line_index += 1
																		continue

												# There is some copied code here, let's retrieve the original.
												A__					,					A__					,					A__  :					Dict											=					search.groups()
												A__  :					Tuple											=					find_code_in_diffusers(UpperCAmelCase__ )
												A__  :					int											=					get_indent(UpperCAmelCase__ )

												A__  :					List[str]											=					line_index + 1 if indent == theoretical_indent else line_index + 2
												A__  :					Tuple											=					theoretical_indent
												A__  :					Optional[Any]											=					start_index

												# Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
												A__  :					Tuple											=					True
												while line_index < len(UpperCAmelCase__ ) and should_continue:
																		line_index += 1
																		if line_index >= len(UpperCAmelCase__ ):
																								break
																		A__  :					Optional[int]											=					lines[line_index]
																		A__  :					Tuple											=					_should_continue(UpperCAmelCase__,  UpperCAmelCase__ ) and re.search(f'^{indent}# End copy',  UpperCAmelCase__ ) is None
												# Clean up empty lines at the end (if any).
												while len(lines[line_index - 1] ) <= 1:
																		line_index -= 1

												A__  :					Dict											=					lines[start_index:line_index]
												A__  :					Tuple											=					"""""".join(UpperCAmelCase__ )

												# Remove any nested `Copied from` comments to avoid circular copies
												A__  :					Optional[int]											=					[line for line in theoretical_code.split("""\n""" ) if _re_copy_warning.search(UpperCAmelCase__ ) is None]
												A__  :					Optional[Any]											=					"""\n""".join(UpperCAmelCase__ )

												# Before comparing, use the `replace_pattern` on the original code.
												if len(UpperCAmelCase__ ) > 0:
																		A__  :					int											=					replace_pattern.replace("""with""",  """""" ).split(""",""" )
																		A__  :					List[Any]											=					[_re_replace_pattern.search(UpperCAmelCase__ ) for p in patterns]
																		for pattern in patterns:
																								if pattern is None:
																														continue
																								A__					,					A__					,					A__  :					Union[str, Any]											=					pattern.groups()
																								A__  :					Union[str, Any]											=					re.sub(UpperCAmelCase__,  UpperCAmelCase__,  UpperCAmelCase__ )
																								if option.strip() == "all-casing":
																														A__  :					List[Any]											=					re.sub(obja.lower(),  obja.lower(),  UpperCAmelCase__ )
																														A__  :					Tuple											=					re.sub(obja.upper(),  obja.upper(),  UpperCAmelCase__ )

            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
																		A__  :					Optional[int]											=					blackify(lines[start_index - 1] + theoretical_code )
																		A__  :					List[Any]											=					theoretical_code[len(lines[start_index - 1] ) :]

												# Test for a diff and act accordingly.
												if observed_code != theoretical_code:
																		diffs.append([object_name, start_index] )
																		if overwrite:
																								A__  :					List[Any]											=					lines[:start_index] + [theoretical_code] + lines[line_index:]
																								A__  :					Tuple											=					start_index + 1

						if overwrite and len(UpperCAmelCase__ ) > 0:
												# Warn the user a file has been modified.
												print(f'Detected changes, rewriting {filename}.' )
												with open(UpperCAmelCase__,  """w""",  encoding="""utf-8""",  newline="""\n""" ) as f:
																		f.writelines(UpperCAmelCase__ )
						return diffs




def        _lowerCAmelCase						(							UpperCAmelCase__    :   bool = False )						->Any:
						A__  :					Dict											=					glob.glob(os.path.join(UpperCAmelCase__,  """**/*.py""" ),  recursive=UpperCAmelCase__ )
						A__  :					str											=					[]
						for filename in all_files:
												A__  :					Any											=					is_copy_consistent(UpperCAmelCase__,  UpperCAmelCase__ )
												diffs += [f'- {filename}: copy does not match {d[0]} at line {d[1]}' for d in new_diffs]
						if not overwrite and len(UpperCAmelCase__ ) > 0:
												A__  :					Any											=					"""\n""".join(UpperCAmelCase__ )
												raise Exception(
												    """Found the following copy inconsistencies:\n"""
												    + diff
												    + """\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them.""" )


if __name__ == "__main__":
						A_      =							argparse.ArgumentParser()
						parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
						A_      =							parser.parse_args()

						check_copies(args.fix_and_overwrite)






