



'''simple docstring'''



import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCAmelCase_  :							Any             =    'http://www.mocksite.com/file1.txt'
lowerCAmelCase_  :							List[str]             =    '"text": ["foo", "foo"]'
lowerCAmelCase_  :							Any             =    '6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'
class 						lowerCamelCase_				:
						_lowerCAmelCase  : Dict						  =					2_0_0
						_lowerCAmelCase  : int						  =					{'Content-Length': '100'}
						_lowerCAmelCase  : Optional[Any]						  =					{}







						def        __lowercase     (	self :   str					,     **lowerCAmelCase__ :   Any					):

													"""simple docstring"""
													return [bytes(lowerCAmelCase__					,     '''utf-8'''					)]


def        UpperCAmelCase (    *A	:					int       ,		**A	:					Optional[Any]					):
							return MockResponse()
@pytest.mark.parametrize('''urls_type'''       ,		[str, list, dict]					)
def        UpperCAmelCase (    A	:					int       ,		A	:					Union[str, Any]       ,		A	:					Union[str, Any]					):
							import requests

							monkeypatch.setattr(A       ,		'''request'''       ,		A					)

							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      URL
							if issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      url
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      [url]
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   str         =      {'''train''': url}
							SCREAMING_SNAKE_CASE       :   Any         =      '''dummy'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      '''downloads'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      tmp_path
							SCREAMING_SNAKE_CASE       :   List[str]         =      DownloadConfig(
							    cache_dir=os.path.join(A       ,		A					)       ,		use_etag=A       ,		)
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      DownloadManager(dataset_name=A       ,		download_config=A					)
							SCREAMING_SNAKE_CASE       :   List[str]         =      dl_manager.download(A					)
							SCREAMING_SNAKE_CASE       :   Dict         =      urls
							for downloaded_paths in [downloaded_paths]:
														if isinstance(A       ,		A					):
																					SCREAMING_SNAKE_CASE       :   List[str]         =      [downloaded_paths]
																					SCREAMING_SNAKE_CASE       :   int         =      [urls]
														elif isinstance(A       ,		A					):
																					assert "train" in downloaded_paths.keys()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      downloaded_paths.values()
																					SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      urls.values()
														assert downloaded_paths
														for downloaded_path, input_url in zip(A       ,		A					):
																					assert downloaded_path == dl_manager.downloaded_paths[input_url]
																					SCREAMING_SNAKE_CASE       :   Dict         =      Path(A					)
																					SCREAMING_SNAKE_CASE       :   Optional[Any]         =      downloaded_path.parts
																					assert parts[-1] == HASH
																					assert parts[-2] == cache_subdir
																					assert downloaded_path.exists()
																					SCREAMING_SNAKE_CASE       :   str         =      downloaded_path.read_text()
																					assert content == CONTENT
																					SCREAMING_SNAKE_CASE       :   str         =      downloaded_path.with_suffix('''.json'''					)
																					assert metadata_downloaded_path.exists()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      json.loads(metadata_downloaded_path.read_text()					)
																					assert metadata_content == {"url": URL, "etag": None}
@pytest.mark.parametrize('''paths_type'''       ,		[str, list, dict]					)
def        UpperCAmelCase (    A	:					Tuple       ,		A	:					Union[str, Any]       ,		A	:					Dict					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      str(A					)
							if issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      filename
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      [filename]
							elif issubclass(A       ,		A					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      {'''train''': filename}
							SCREAMING_SNAKE_CASE       :   Dict         =      '''dummy'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      xz_file.parent
							SCREAMING_SNAKE_CASE       :   List[Any]         =      '''extracted'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      DownloadConfig(
							    cache_dir=A       ,		use_etag=A       ,		)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      DownloadManager(dataset_name=A       ,		download_config=A					)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      dl_manager.extract(A					)
							SCREAMING_SNAKE_CASE       :   Tuple         =      paths
							for extracted_paths in [extracted_paths]:
														if isinstance(A       ,		A					):
																					SCREAMING_SNAKE_CASE       :   List[Any]         =      [extracted_paths]
																					SCREAMING_SNAKE_CASE       :   str         =      [paths]
														elif isinstance(A       ,		A					):
																					assert "train" in extracted_paths.keys()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      extracted_paths.values()
																					SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      paths.values()
														assert extracted_paths
														for extracted_path, input_path in zip(A       ,		A					):
																					assert extracted_path == dl_manager.extracted_paths[input_path]
																					SCREAMING_SNAKE_CASE       :   Any         =      Path(A					)
																					SCREAMING_SNAKE_CASE       :   Optional[Any]         =      extracted_path.parts
																					assert parts[-1] == hash_url_to_filename(A       ,		etag=A					)
																					assert parts[-2] == extracted_subdir
																					assert extracted_path.exists()
																					SCREAMING_SNAKE_CASE       :   int         =      extracted_path.read_text()
																					SCREAMING_SNAKE_CASE       :   Optional[int]         =      text_file.read_text()
																					assert extracted_file_content == expected_file_content
def        UpperCAmelCase (    A	:					Optional[int]       ,		A	:					int					):
							assert path.endswith('''.jsonl'''					)
							for num_items, line in enumerate(A       ,		start=1					):
														SCREAMING_SNAKE_CASE       :   List[Any]         =      json.loads(line.decode('''utf-8'''					)					)
														assert item.keys() == {"col_1", "col_2", "col_3"}
							assert num_items == 4
@pytest.mark.parametrize('''archive_jsonl'''       ,		['''tar_jsonl_path''', '''zip_jsonl_path''']					)
def        UpperCAmelCase (    A	:					Any       ,		A	:					str					):
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      request.getfixturevalue(A					)
							SCREAMING_SNAKE_CASE       :   List[Any]         =      DownloadManager()
							for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
														_test_jsonl(A       ,		A					)
							assert num_jsonl == 2
@pytest.mark.parametrize('''archive_nested_jsonl'''       ,		['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']					)
def        UpperCAmelCase (    A	:					Dict       ,		A	:					Optional[Any]					):
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      request.getfixturevalue(A					)
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      DownloadManager()
							for num_tar, (path, file) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
														for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(A					)       ,		start=1					):
																					_test_jsonl(A       ,		A					)
							assert num_tar == 1
							assert num_jsonl == 2






def        UpperCAmelCase (    A	:					List[Any]					):
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      DownloadManager()
							for num_file, file in enumerate(dl_manager.iter_files(A					)       ,		start=1					):
														assert os.path.basename(A					) == ("test.txt" if num_file == 1 else "train.txt")
							assert num_file == 2









'''simple docstring'''



import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import DonutImageProcessor
class 						lowerCamelCase_				(       unittest.TestCase      ):







						def __init__(	self :   Dict					,     lowerCAmelCase__ :   Union[str, Any]					,     lowerCAmelCase__ :   Dict=7					,     lowerCAmelCase__ :   List[str]=3					,     lowerCAmelCase__ :   Union[str, Any]=18					,     lowerCAmelCase__ :   Tuple=30					,     lowerCAmelCase__ :   Any=4_00					,     lowerCAmelCase__ :   Optional[int]=True					,     lowerCAmelCase__ :   List[Any]=None					,     lowerCAmelCase__ :   int=True					,     lowerCAmelCase__ :   Optional[Any]=False					,     lowerCAmelCase__ :   Any=True					,     lowerCAmelCase__ :   Optional[Any]=True					,     lowerCAmelCase__ :   Union[str, Any]=[0.5, 0.5, 0.5]					,     lowerCAmelCase__ :   List[str]=[0.5, 0.5, 0.5]					,     ):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      parent
													SCREAMING_SNAKE_CASE       :   int         =      batch_size
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      num_channels
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      image_size
													SCREAMING_SNAKE_CASE       :   Any         =      min_resolution
													SCREAMING_SNAKE_CASE       :   int         =      max_resolution
													SCREAMING_SNAKE_CASE       :   Tuple         =      do_resize
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      size if size is not None else {'''height''': 18, '''width''': 20}
													SCREAMING_SNAKE_CASE       :   List[Any]         =      do_thumbnail
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      do_align_axis
													SCREAMING_SNAKE_CASE       :   Tuple         =      do_pad
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      do_normalize
													SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      image_mean
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      image_std







						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													return {
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "do_thumbnail": self.do_thumbnail,
													    "do_align_long_axis": self.do_align_axis,
													    "do_pad": self.do_pad,
													    "do_normalize": self.do_normalize,
													    "image_mean": self.image_mean,
													    "image_std": self.image_std,
													}


@require_torch
@require_vision
class 						lowerCamelCase_				(       snake_case_     ,					unittest.TestCase      ):
						_lowerCAmelCase  : Any						  =					DonutImageProcessor if is_vision_available() else None







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Tuple         =      DonutImageProcessingTester(self					)







						@property
						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													return self.image_processor_tester.prepare_image_processor_dict()







						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   str         =      self.image_processing_class(**self.image_processor_dict					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''do_resize'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''size'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''do_thumbnail'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''do_align_long_axis'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''do_pad'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''do_normalize'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''image_mean'''					)					)
													self.assertTrue(hasattr(lowerCAmelCase__					,     '''image_std'''					)					)







						def        __lowercase     (	self :   Any					):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      self.image_processing_class.from_dict(self.image_processor_dict					)
													self.assertEqual(image_processor.size					,     {'''height''': 18, '''width''': 20}					)

													SCREAMING_SNAKE_CASE       :   List[str]         =      self.image_processing_class.from_dict(self.image_processor_dict					,     size=42					)
													self.assertEqual(image_processor.size					,     {'''height''': 42, '''width''': 42}					)

													# Previous config had dimensions in (width, height) order
													SCREAMING_SNAKE_CASE       :   Dict         =      self.image_processing_class.from_dict(self.image_processor_dict					,     size=(42, 84)					)
													self.assertEqual(image_processor.size					,     {'''height''': 84, '''width''': 42}					)







						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													pass







						@is_flaky()
						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													# Initialize image_processing
													SCREAMING_SNAKE_CASE       :   Optional[int]         =      self.image_processing_class(**self.image_processor_dict					)
													# create random PIL images
													SCREAMING_SNAKE_CASE       :   List[str]         =      prepare_image_inputs(self.image_processor_tester					,     equal_resolution=lowerCAmelCase__					)
													for image in image_inputs:
																				self.assertIsInstance(lowerCAmelCase__					,     Image.Image					)

													# Test not batched input
													SCREAMING_SNAKE_CASE       :   Any         =      image_processing(image_inputs[0]					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )

													# Test batched
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      image_processing(lowerCAmelCase__					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )







						@is_flaky()
						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													# Initialize image_processing
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.image_processing_class(**self.image_processor_dict					)
													# create random numpy tensors
													SCREAMING_SNAKE_CASE       :   Tuple         =      prepare_image_inputs(self.image_processor_tester					,     equal_resolution=lowerCAmelCase__					,     numpify=lowerCAmelCase__					)
													for image in image_inputs:
																				self.assertIsInstance(lowerCAmelCase__					,     np.ndarray					)

													# Test not batched input
													SCREAMING_SNAKE_CASE       :   int         =      image_processing(image_inputs[0]					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )

													# Test batched
													SCREAMING_SNAKE_CASE       :   int         =      image_processing(lowerCAmelCase__					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )







						@is_flaky()
						def        __lowercase     (	self :   List[Any]					):

													"""simple docstring"""
													# Initialize image_processing
													SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.image_processing_class(**self.image_processor_dict					)
													# create random PyTorch tensors
													SCREAMING_SNAKE_CASE       :   Tuple         =      prepare_image_inputs(self.image_processor_tester					,     equal_resolution=lowerCAmelCase__					,     torchify=lowerCAmelCase__					)
													for image in image_inputs:
																				self.assertIsInstance(lowerCAmelCase__					,     torch.Tensor					)

													# Test not batched input
													SCREAMING_SNAKE_CASE       :   str         =      image_processing(image_inputs[0]					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )

													# Test batched
													SCREAMING_SNAKE_CASE       :   List[str]         =      image_processing(lowerCAmelCase__					,     return_tensors='''pt'''					).pixel_values
													self.assertEqual(
													    encoded_images.shape					,     (
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.size['''height'''],
													        self.image_processor_tester.size['''width'''],
													    )					,     )




