






'''simple docstring'''





import argparse
import os
import re

import packaging.version


_UpperCamelCase						:						Union[str, Any]  =					'examples/'
_UpperCamelCase						:						List[str]  =					{
    'examples': (re.compile(R'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(R'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(R'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), R'\1version="VERSION",'),
    'doc': (re.compile(R'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
_UpperCamelCase						:						Optional[Any]  =					{
    'init': 'src/diffusers/__init__.py',
    'setup': 'setup.py',
}
_UpperCamelCase						:						Any  =					'README.md'
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			List[Any]			,       A					:			Tuple		)    ->   Dict:
			with open(A			,       '''r'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						UpperCAmelCase_      :       int          =       f.read()
			UpperCAmelCase_    ,      UpperCAmelCase_      :       Union[str, Any]          =       REPLACE_PATTERNS[pattern]
			UpperCAmelCase_      :       Tuple          =       replace.replace('''VERSION'''			,       A		)
			UpperCAmelCase_      :       Tuple          =       re_pattern.sub(A			,       A		)
			with open(A			,       '''w'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						f.write(A		)
def __UpperCAmelCase   (					A					:			List[str]		)    ->   Any:
			for folder, directories, fnames in os.walk(A		):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove('''research_projects'''		)
						if "legacy" in directories:
									directories.remove('''legacy'''		)
						for fname in fnames:
									if fname.endswith('''.py'''		):
												update_version_in_file(os.path.join(A			,       A		)			,       A			,       pattern='''examples'''		)
def __UpperCAmelCase   (					A					:			Tuple			,       A					:			Optional[int]=False		)    ->   Tuple:
			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(A			,       A			,       A		)
			if not patch:
						update_version_in_examples(A		)
def __UpperCAmelCase   (					)    ->   int:
			UpperCAmelCase_      :       List[str]          =       '''🤗 Transformers currently provides the following architectures'''
			UpperCAmelCase_      :       int          =       '''1. Want to contribute a new model?'''
			with open(A			,       '''r'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						UpperCAmelCase_      :       Optional[int]          =       f.readlines()

			# Find the start of the list.
			UpperCAmelCase_      :       List[str]          =       0
			while not lines[start_index].startswith(_start_prompt		):
						start_index += 1
			start_index += 1

			UpperCAmelCase_      :       Any          =       start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt		):
						if lines[index].startswith('''1.'''		):
									UpperCAmelCase_      :       Optional[Any]          =       lines[index].replace(
									    '''https://huggingface.co/docs/diffusers/main/model_doc'''			,       '''https://huggingface.co/docs/diffusers/model_doc'''			,       )
						index += 1

			with open(A			,       '''w'''			,       encoding='''utf-8'''			,       newline='''\n'''		) as f:
						f.writelines(A		)
def __UpperCAmelCase   (					)    ->   Dict:
			with open(REPLACE_FILES['''init''']			,       '''r'''		) as f:
						UpperCAmelCase_      :       Any          =       f.read()
			UpperCAmelCase_      :       Union[str, Any]          =       REPLACE_PATTERNS['''init'''][0].search(A		).groups()[0]
			return packaging.version.parse(A		)
def __UpperCAmelCase   (					A					:			str=False		)    ->   List[Any]:
			UpperCAmelCase_      :       Union[str, Any]          =       get_version()
			if patch and default_version.is_devrelease:
						raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''		)
			if default_version.is_devrelease:
						UpperCAmelCase_      :       Optional[int]          =       default_version.base_version
			elif patch:
						UpperCAmelCase_      :       Any          =       F"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
			else:
						UpperCAmelCase_      :       Optional[Any]          =       F"{default_version.major}.{default_version.minor + 1}.0"

			# Now let's ask nicely if that's the right one.
			UpperCAmelCase_      :       List[str]          =       input(F"Which version are you releasing? [{default_version}]"		)
			if len(A		) == 0:
						UpperCAmelCase_      :       Tuple          =       default_version

			print(F"Updating version to {version}."		)
			global_version_update(A			,       patch=A		)
def __UpperCAmelCase   (					)    ->   Optional[int]:
			UpperCAmelCase_      :       str          =       get_version()
			UpperCAmelCase_      :       Tuple          =       F"{current_version.major}.{current_version.minor + 1}.0.dev0"
			UpperCAmelCase_      :       Any          =       current_version.base_version

			# Check with the user we got that right.
			UpperCAmelCase_      :       List[Any]          =       input(F"Which version are we developing now? [{dev_version}]"		)
			if len(A		) == 0:
						UpperCAmelCase_      :       Optional[int]          =       dev_version

			print(F"Updating version to {version}."		)
			global_version_update(A		)


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
						_UpperCamelCase						:						Tuple  =					argparse.ArgumentParser()
						parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
						parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
						_UpperCamelCase						:						Optional[Any]  =					parser.parse_args()
						if not args.post_release:
												pre_release_work(patch=args.patch)
						elif args.patch:
												print('Nothing to do after a patch :-)')
						else:
												post_release_work()










'''simple docstring'''





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_UpperCamelCase						:						str  =					datasets.utils.logging.get_logger(__name__)


@dataclass
class  snake_case__      (					datasets.BuilderConfig):
					a_        =					None
					a_        =					"utf-8"
					a_        =					None
					a_        =					None
					a_        =					True  # deprecated
					a_        =					None  # deprecated
					a_        =					10 << 20  # 10MB
					a_        =					None






class  snake_case__      (					datasets.ArrowBasedBuilder):
					a_        =					JsonConfig




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								if self.config.block_size is not None:
											logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
											UpperCAmelCase_      :       Union[str, Any]          =       self.config.block_size
								if self.config.use_threads is not True:
											logger.warning(
											    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
								if self.config.newlines_in_values is not None:
											raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
								return datasets.DatasetInfo(features=self.config.features      )




					def 		A      (  self  :		Optional[int]				, _A  :		List[str]      )    ->							int:
								if not self.config.data_files:
											raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"      )
								UpperCAmelCase_      :       Optional[Any]          =       dl_manager.download_and_extract(self.config.data_files      )
								if isinstance(_A				, (str, list, tuple)      ):
											UpperCAmelCase_      :       List[Any]          =       data_files
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       List[str]          =       [files]
											UpperCAmelCase_      :       List[str]          =       [dl_manager.iter_files(_A      ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''files''': files}      )]
								UpperCAmelCase_      :       Tuple          =       []
								for split_name, files in data_files.items():
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       Optional[Any]          =       [files]
											UpperCAmelCase_      :       List[Any]          =       [dl_manager.iter_files(_A      ) for file in files]
											splits.append(datasets.SplitGenerator(name=_A				, gen_kwargs={'''files''': files}      )      )
								return splits




					def 		A      (  self  :		List[Any]				, _A  :		pa.Table      )    ->							pa.Table:
								if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
														UpperCAmelCase_      :       Union[str, Any]          =       self.config.features.arrow_schema.field(_A      ).type
														UpperCAmelCase_      :       int          =       pa_table.append_column(_A				, pa.array([None] * len(_A      )				, type=_A      )      )
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											UpperCAmelCase_      :       Dict          =       table_cast(_A				, self.config.features.arrow_schema      )
								return pa_table





					def 		A      (  self  :		Optional[Any]				, _A  :		List[Any]      )    ->							Optional[int]:
								for file_idx, file in enumerate(itertools.chain.from_iterable(_A      )      ):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
														with open(_A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																	UpperCAmelCase_      :       Optional[int]          =       json.load(_A      )

														# We keep only the field we are interested in
														UpperCAmelCase_      :       Any          =       dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(_A				, (list, tuple)      ):
																	UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																	UpperCAmelCase_      :       str          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
														else:
																	UpperCAmelCase_      :       Tuple          =       dataset
														UpperCAmelCase_      :       List[str]          =       pa.Table.from_pydict(_A      )
														yield file_idx, self._cast_table(_A      )

											# If the file has one json object per line
											else:
														with open(_A				, '''rb'''      ) as f:
																	UpperCAmelCase_      :       Tuple          =       0
																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																	# Set a default minimum value of 16kB if the chunk size is really small
																	UpperCAmelCase_      :       Optional[Any]          =       max(self.config.chunksize // 32				, 16 << 10      )
																	UpperCAmelCase_      :       Tuple          =       (
																	    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																	)
																	while True:
																				UpperCAmelCase_      :       Optional[Any]          =       f.read(self.config.chunksize      )
																				if not batch:
																							break
																				# Finish current line
																				try:
																							batch += f.readline()
																				except (AttributeError, io.UnsupportedOperation):
																							batch += readline(_A      )
																				# PyArrow only accepts utf-8 encoded bytes
																				if self.config.encoding != "utf-8":
																							UpperCAmelCase_      :       List[Any]          =       batch.decode(self.config.encoding				, errors=_A      ).encode('''utf-8'''      )
																				try:
																							while True:
																										try:
																													UpperCAmelCase_      :       int          =       paj.read_json(
																													    io.BytesIO(_A      )				, read_options=paj.ReadOptions(block_size=_A      )      )
																													break
																										except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																													if (
																													    isinstance(_A				, pa.ArrowInvalid      )
																													    and "straddling" not in str(_A      )
																													    or block_size > len(_A      )
																													):
																																raise
																													else:
																																# Increase the block size in case it was too small.
																																# The block size will be reset for the next file.
																																logger.debug(
																																    F"Batch of {len(_A      )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."      )
																																block_size *= 2
																				except pa.ArrowInvalid as e:
																							try:
																										with open(
																										    _A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																													UpperCAmelCase_      :       List[Any]          =       json.load(_A      )
																							except json.JSONDecodeError:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise e
																							# If possible, parse the file as a list of json objects and exit the loop
																							if isinstance(_A				, _A      ):  # list is the only sequence type supported in JSON
																										try:
																													UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																													UpperCAmelCase_      :       Any          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
																													UpperCAmelCase_      :       Optional[int]          =       pa.Table.from_pydict(_A      )
																										except (pa.ArrowInvalid, AttributeError) as e:
																													logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																													raise ValueError(F"Not able to read records in the JSON file at {file}."      ) from None
																										yield file_idx, self._cast_table(_A      )
																										break
																							else:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise ValueError(
																										    F"Not able to read records in the JSON file at {file}. "
																										    F"You should probably indicate the field of the JSON file containing your records. "
																										    F"This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. "
																										    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																				yield (file_idx, batch_idx), self._cast_table(_A      )
																				batch_idx += 1


