



import bza
import gzip
import lzma
import os
import shutil
import struct
import tarfile
import warnings
import zipfile
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Dict, List, Optional, Type, Union

from .. import config
from .filelock import FileLock
from .logging import get_logger


lowerCamelCase__			= get_logger(__name__)






class    SCREAMING_SNAKE_CASE			:





							def __init__(   self		:		Optional[Any]      ,				__lowercase		:		Optional[str] = None	):

											'''simple docstring'''
											__a       =		(
											    os.path.join(__lowercase      ,				config.EXTRACTED_DATASETS_DIR	) if cache_dir else config.EXTRACTED_DATASETS_PATH
											)
											__a       =		Extractor





							def 						UpperCamelCase_					(   self		:		str      ,				__lowercase		:		str	):

											'''simple docstring'''
											from .file_utils import hash_url_to_filename

											# Path where we extract compressed archives
											# We extract in the cache dir, and get the extracted path name by hashing the original path"
											__a       =		os.path.abspath(__lowercase	)
											return os.path.join(self.extract_dir      ,				hash_url_to_filename(__lowercase	)	)





							def 						UpperCamelCase_					(   self		:		Union[str, Any]      ,				__lowercase		:		str      ,				__lowercase		:		bool	):

											'''simple docstring'''
											return force_extract or (
											    not os.path.isfile(__lowercase	) and not (os.path.isdir(__lowercase	) and os.listdir(__lowercase	))
											)







							def 						UpperCamelCase_					(   self		:		List[Any]      ,				__lowercase		:		str      ,				__lowercase		:		bool = False	):

											'''simple docstring'''
											__a       =		self.extractor.infer_extractor_format(__lowercase	)
											if not extractor_format:
															return input_path
											__a       =		self._get_output_path(__lowercase	)
											if self._do_extract(__lowercase      ,				__lowercase	):
															self.extractor.extract(__lowercase      ,				__lowercase      ,				__lowercase	)
											return output_path






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):





							@classmethod
							@abstractmethod
							def 						UpperCamelCase_					(   cls		:		List[str]      ,				__lowercase		:		Union[Path, str]      ,				**__lowercase		:		List[str]	):

											'''simple docstring'''
											...







							@staticmethod
							@abstractmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											...






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__			,      lowerCamelCase__					):
							__lowerCamelCase	:	List[bytes]    =[]





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		int	):

											'''simple docstring'''
											with open(__lowercase      ,				"""rb"""	) as f:
															return f.read(__lowercase	)







							@classmethod
							def 						UpperCamelCase_					(   cls		:		Tuple      ,				__lowercase		:		Union[Path, str]      ,				__lowercase		:		bytes = b""	):

											'''simple docstring'''
											if not magic_number:
															__a       =		max(len(__lowercase	) for cls_magic_number in cls.magic_numbers	)
															try:
																			__a       =		cls.read_magic_number(__lowercase      ,				__lowercase	)
															except OSError:
																			return False
											return any(magic_number.startswith(__lowercase	) for cls_magic_number in cls.magic_numbers	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):





							@classmethod
							def 						UpperCamelCase_					(   cls		:		Tuple      ,				__lowercase		:		Union[Path, str]      ,				**__lowercase		:		int	):

											'''simple docstring'''
											return tarfile.is_tarfile(__lowercase	)





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Dict      ,				__lowercase		:		List[Any]	):

											'''simple docstring'''

											def resolved(__lowercase		:		str	) -> str:
															return os.path.realpath(os.path.abspath(__lowercase	)	)

											def badpath(__lowercase		:		str      ,				__lowercase		:		str	) -> bool:
															# joinpath will ignore base if path is absolute
															return not resolved(os.path.join(__lowercase      ,				__lowercase	)	).startswith(__lowercase	)

											def badlink(__lowercase		:		Union[str, Any]      ,				__lowercase		:		str	) -> bool:
															# Links are interpreted relative to the directory containing the link
															__a       =		resolved(os.path.join(__lowercase      ,				os.path.dirname(info.name	)	)	)
															return badpath(info.linkname      ,				base=__lowercase	)

											__a       =		resolved(__lowercase	)

											for finfo in members:
															if badpath(finfo.name      ,				__lowercase	):
																			logger.error(F"Extraction of {finfo.name} is blocked (illegal path)"	)
															elif finfo.issym() and badlink(__lowercase      ,				__lowercase	):
																			logger.error(F"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}"	)
															elif finfo.islnk() and badlink(__lowercase      ,				__lowercase	):
																			logger.error(F"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}"	)
															else:
																			yield finfo







							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											os.makedirs(__lowercase      ,				exist_ok=__lowercase	)
											__a       =		tarfile.open(__lowercase	)
											tar_file.extractall(__lowercase      ,				members=TarExtractor.safemembers(__lowercase      ,				__lowercase	)	)
											tar_file.close()






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Any    =[b'\x1F\x8B']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											with gzip.open(__lowercase      ,				"""rb"""	) as gzip_file:
															with open(__lowercase      ,				"""wb"""	) as extracted_file:
																			shutil.copyfileobj(__lowercase      ,				__lowercase	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Any    =[
							    b'PK\x03\x04',
							    b'PK\x05\x06',  # empty archive
							    b'PK\x07\x08',  # spanned archive
							]





							@classmethod
							def 						UpperCamelCase_					(   cls		:		Dict      ,				__lowercase		:		Union[Path, str]      ,				__lowercase		:		bytes = b""	):

											'''simple docstring'''
											if super().is_extractable(__lowercase      ,				magic_number=__lowercase	):
															return True
											try:
															# Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives.
															# From: https://github.com/python/cpython/pull/5053
															from zipfile import (
															    _CD_SIGNATURE,
															    _ECD_DISK_NUMBER,
															    _ECD_DISK_START,
															    _ECD_ENTRIES_TOTAL,
															    _ECD_OFFSET,
															    _ECD_SIZE,
															    _EndRecData,
															    sizeCentralDir,
															    stringCentralDir,
															    structCentralDir,
															)

															with open(__lowercase      ,				"""rb"""	) as fp:
																			__a       =		_EndRecData(__lowercase	)
																			if endrec:
																							if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0:
																											return True  # Empty zipfiles are still zipfiles
																							elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]:
																											fp.seek(endrec[_ECD_OFFSET]	)  # Central directory is on the same disk
																											if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir:
																															__a       =		fp.read(__lowercase	)  # CD is where we expect it to be
																															if len(__lowercase	) == sizeCentralDir:
																																			__a       =		struct.unpack(__lowercase      ,				__lowercase	)  # CD is the right size
																																			if centdir[_CD_SIGNATURE] == stringCentralDir:
																																							return True  # First central directory entry  has correct magic number
															return False
											except Exception:  # catch all errors in case future python versions change the zipfile internals
															return False







							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											os.makedirs(__lowercase      ,				exist_ok=__lowercase	)
											with zipfile.ZipFile(__lowercase      ,				"""r"""	) as zip_file:
															zip_file.extractall(__lowercase	)
															zip_file.close()






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Tuple    =[b'\xFD\x37\x7A\x58\x5A\x00']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											with lzma.open(__lowercase	) as compressed_file:
															with open(__lowercase      ,				"""wb"""	) as extracted_file:
																			shutil.copyfileobj(__lowercase      ,				__lowercase	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Dict    =[b'Rar!\x1a\x07\x00', b'Rar!\x1a\x07\x01\x00']  # RAR_ID  # RAR5_ID





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											if not config.RARFILE_AVAILABLE:
															raise ImportError("""Please pip install rarfile"""	)
											import rarfile

											os.makedirs(__lowercase      ,				exist_ok=__lowercase	)
											__a       =		rarfile.RarFile(__lowercase	)
											rf.extractall(__lowercase	)
											rf.close()






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Dict    =[b'\x28\xb5\x2F\xFD']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											if not config.ZSTANDARD_AVAILABLE:
															raise ImportError("""Please pip install zstandard"""	)
											import zstandard as zstd

											__a       =		zstd.ZstdDecompressor()
											with open(__lowercase      ,				"""rb"""	) as ifh, open(__lowercase      ,				"""wb"""	) as ofh:
															dctx.copy_stream(__lowercase      ,				__lowercase	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Optional[int]    =[b'\x42\x5A\x68']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											with bza.open(__lowercase      ,				"""rb"""	) as compressed_file:
															with open(__lowercase      ,				"""wb"""	) as extracted_file:
																			shutil.copyfileobj(__lowercase      ,				__lowercase	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Optional[Any]    =[b'\x37\x7A\xBC\xAF\x27\x1C']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											if not config.PY7ZR_AVAILABLE:
															raise ImportError("""Please pip install py7zr"""	)
											import pyazr

											os.makedirs(__lowercase      ,				exist_ok=__lowercase	)
											with pyazr.SevenZipFile(__lowercase      ,				"""r"""	) as archive:
															archive.extractall(__lowercase	)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	Optional[int]    =[b'\x04\x22\x4D\x18']





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]	):

											'''simple docstring'''
											if not config.LZ4_AVAILABLE:
															raise ImportError("""Please pip install lz4"""	)
											import lza.frame

											with lza.frame.open(__lowercase      ,				"""rb"""	) as compressed_file:
															with open(__lowercase      ,				"""wb"""	) as extracted_file:
																			shutil.copyfileobj(__lowercase      ,				__lowercase	)






class    SCREAMING_SNAKE_CASE			:
							#  Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip)
							__lowerCamelCase	:	Dict[str, Type[BaseExtractor]]    ={
							    "tar": TarExtractor,
							    "gzip": GzipExtractor,
							    "zip": ZipExtractor,
							    "xz": XzExtractor,
							    "rar": RarExtractor,
							    "zstd": ZstdExtractor,
							    "bz2": BzipaExtractor,
							    "7z": SevenZipExtractor,  # <Added version="2.4.0"/>
							    "lz4": LzaExtractor,  # <Added version="2.4.0"/>
							}





							@classmethod
							def 						UpperCamelCase_					(   cls		:		str	):

											'''simple docstring'''
											return max(
											    len(__lowercase	)
											    for extractor in cls.extractors.values()
											    if issubclass(__lowercase      ,				__lowercase	)
											    for extractor_magic_number in extractor.magic_numbers	)





							@staticmethod
							def 						UpperCamelCase_					(   __lowercase		:		Union[Path, str]      ,				__lowercase		:		int	):

											'''simple docstring'''
											try:
															return MagicNumberBaseExtractor.read_magic_number(__lowercase      ,				magic_number_length=__lowercase	)
											except OSError:
															return b""





							@classmethod
							def 						UpperCamelCase_					(   cls		:		Tuple      ,				__lowercase		:		Union[Path, str]      ,				__lowercase		:		bool = False	):

											'''simple docstring'''
											warnings.warn(
											    """Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. """
											    """Use 'infer_extractor_format' instead."""      ,				category=__lowercase      ,				)
											__a       =		cls.infer_extractor_format(__lowercase	)
											if extractor_format:
															return True if not return_extractor else (True, cls.extractors[extractor_format])
											return False if not return_extractor else (False, None)





							@classmethod
							def 						UpperCamelCase_					(   cls		:		Optional[int]      ,				__lowercase		:		Union[Path, str]	):  # <Added version="2.4.0"/>

											'''simple docstring'''
											__a       =		cls._get_magic_number_max_length()
											__a       =		cls._read_magic_number(__lowercase      ,				__lowercase	)
											for extractor_format, extractor in cls.extractors.items():
															if extractor.is_extractable(__lowercase      ,				magic_number=__lowercase	):
																			return extractor_format







							@classmethod
							def 						UpperCamelCase_					(   cls		:		Optional[int]      ,				__lowercase		:		Union[Path, str]      ,				__lowercase		:		Union[Path, str]      ,				__lowercase		:		Optional[str] = None      ,				__lowercase		:		Optional[BaseExtractor] = "deprecated"      ,				):

											'''simple docstring'''
											os.makedirs(os.path.dirname(__lowercase	)      ,				exist_ok=__lowercase	)
											# Prevent parallel extractions
											__a       =		str(Path(__lowercase	).with_suffix(""".lock"""	)	)
											with FileLock(__lowercase	):
															shutil.rmtree(__lowercase      ,				ignore_errors=__lowercase	)
															if extractor_format or extractor != "deprecated":
																			if extractor != "deprecated" or not isinstance(__lowercase      ,				__lowercase	):  # passed as positional arg
																							warnings.warn(
																							    """Parameter 'extractor' was deprecated in version 2.4.0 and will be removed in 3.0.0. """
																							    """Use 'extractor_format' instead."""      ,				category=__lowercase      ,				)
																							__a       =		extractor if extractor != """deprecated""" else extractor_format
																			else:
																							__a       =		cls.extractors[extractor_format]
																			return extractor.extract(__lowercase      ,				__lowercase	)
															else:
																			warnings.warn(
																			    """Parameter 'extractor_format' was made required in version 2.4.0 and not passing it will raise an """
																			    """exception in 3.0.0."""      ,				category=__lowercase      ,				)
																			for extractor in cls.extractors.values():
																							if extractor.is_extractable(__lowercase	):
																											return extractor.extract(__lowercase      ,				__lowercase	)




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


lowerCamelCase__			= logging.get_logger(__name__)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	str    =['pixel_values']





							def __init__(   self		:		str      ,				__lowercase		:		bool = True      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		PILImageResampling = PIL.Image.BICUBIC      ,				__lowercase		:		bool = True      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		Union[int, float] = 1 / 255      ,				__lowercase		:		bool = True      ,				__lowercase		:		bool = True      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				**__lowercase		:		Any      ,				):

											'''simple docstring'''
											super().__init__(**__lowercase	)
											__a       =		size if size is not None else {"""height""": 256, """width""": 256}
											__a       =		get_size_dict(__lowercase	)
											__a       =		crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
											__a       =		get_size_dict(__lowercase      ,				param_name="""crop_size"""	)

											__a       =		do_resize
											__a       =		size
											__a       =		resample
											__a       =		do_center_crop
											__a       =		crop_size
											__a       =		do_rescale
											__a       =		rescale_factor
											__a       =		do_normalize
											__a       =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
											__a       =		image_std if image_std is not None else IMAGENET_STANDARD_STD





							def 						UpperCamelCase_					(   self		:		List[Any]      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Dict[str, int]      ,				__lowercase		:		PILImageResampling = PIL.Image.BICUBIC      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		Union[str, Any]      ,				):

											'''simple docstring'''
											__a       =		get_size_dict(__lowercase	)
											if "height" not in size or "width" not in size:
															raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"	)
											return resize(
											    __lowercase      ,				size=(size["""height"""], size["""width"""])      ,				resample=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		Dict      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Dict[str, int]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		str      ,				):

											'''simple docstring'''
											__a       =		get_size_dict(__lowercase	)
											if "height" not in size or "width" not in size:
															raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"	)
											return center_crop(__lowercase      ,				size=(size["""height"""], size["""width"""])      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		int      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Union[int, float]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		List[str]      ,				):

											'''simple docstring'''
											return rescale(__lowercase      ,				scale=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		Tuple      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Union[float, List[float]]      ,				__lowercase		:		Union[float, List[float]]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		str      ,				):

											'''simple docstring'''
											return normalize(__lowercase      ,				mean=__lowercase      ,				std=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)







							def 						UpperCamelCase_					(   self		:		Optional[Any]      ,				__lowercase		:		ImageInput      ,				__lowercase		:		bool = None      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		Dict=None      ,				__lowercase		:		bool = None      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		bool = None      ,				__lowercase		:		float = None      ,				__lowercase		:		bool = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[str, TensorType]] = None      ,				__lowercase		:		ChannelDimension = ChannelDimension.FIRST      ,				**__lowercase		:		Optional[Any]      ,				):

											'''simple docstring'''
											__a       =		do_resize if do_resize is not None else self.do_resize
											__a       =		resample if resample is not None else self.resample
											__a       =		do_center_crop if do_center_crop is not None else self.do_center_crop
											__a       =		do_rescale if do_rescale is not None else self.do_rescale
											__a       =		rescale_factor if rescale_factor is not None else self.rescale_factor
											__a       =		do_normalize if do_normalize is not None else self.do_normalize
											__a       =		image_mean if image_mean is not None else self.image_mean
											__a       =		image_std if image_std is not None else self.image_std

											__a       =		size if size is not None else self.size
											__a       =		get_size_dict(__lowercase	)
											__a       =		crop_size if crop_size is not None else self.crop_size
											__a       =		get_size_dict(__lowercase      ,				param_name="""crop_size"""	)

											__a       =		make_list_of_images(__lowercase	)

											if not valid_images(__lowercase	):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""	)

											if do_resize and size is None or resample is None:
															raise ValueError("""Size and resample must be specified if do_resize is True."""	)

											if do_center_crop and crop_size is None:
															raise ValueError("""Crop size must be specified if do_center_crop is True."""	)

											if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""	)

											if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("""Image mean and std must be specified if do_normalize is True."""	)

											# All transformations expect numpy arrays.
											__a       =		[to_numpy_array(__lowercase	) for image in images]

											if do_resize:
															__a       =		[self.resize(image=__lowercase      ,				size=__lowercase      ,				resample=__lowercase	) for image in images]

											if do_center_crop:
															__a       =		[self.center_crop(image=__lowercase      ,				size=__lowercase	) for image in images]

											if do_rescale:
															__a       =		[self.rescale(image=__lowercase      ,				scale=__lowercase	) for image in images]

											if do_normalize:
															__a       =		[self.normalize(image=__lowercase      ,				mean=__lowercase      ,				std=__lowercase	) for image in images]

											__a       =		[to_channel_dimension_format(__lowercase      ,				__lowercase	) for image in images]

											__a       =		{"""pixel_values""": images}
											return BatchFeature(data=__lowercase      ,				tensor_type=__lowercase	)
