
'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
					import PIL


__lowercase							:   str					=     logging.get_logger(__name__)

class 	__lowercase				(  _lowercase    ):
			lowerCamelCase			:		Dict	      =			["pixel_values"]

			def __init__(self			,   A = True			,   A = None			,   A = PILImageResampling.BICUBIC			,   A = True			,   A = 1 / 2_5_5			,   A = True			,   A = None			,   A = None			,   A = True			,   **A			,   ):
								super().__init__(**A      )
								lowerCamelCase_					:      Dict     =		size if size is not None else {'''height''': 3_8_4, '''width''': 3_8_4}
								lowerCamelCase_					:      Optional[Any]     =		get_size_dict(A			,   default_to_square=A      )

								lowerCamelCase_					:      Dict     =		do_resize
								lowerCamelCase_					:      Optional[int]     =		size
								lowerCamelCase_					:      str     =		resample
								lowerCamelCase_					:      int     =		do_rescale
								lowerCamelCase_					:      Optional[int]     =		rescale_factor
								lowerCamelCase_					:      List[str]     =		do_normalize
								lowerCamelCase_					:      List[Any]     =		image_mean if image_mean is not None else OPENAI_CLIP_MEAN
								lowerCamelCase_					:      Optional[int]     =		image_std if image_std is not None else OPENAI_CLIP_STD
								lowerCamelCase_					:      Tuple     =		do_convert_rgb

			def        UpperCAmelCase__ (self			,   A			,   A			,   A = PILImageResampling.BICUBIC			,   A = None			,   **A			,   ):
								lowerCamelCase_					:      Any     =		get_size_dict(A			,   default_to_square=A      )
								if "height" not in size or "width" not in size:
													raise ValueError(F"""The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}"""      )
								lowerCamelCase_					:      str     =		(size['''height'''], size['''width'''])
								return resize(A			,   size=A			,   resample=A			,   data_format=A			,   **A      )

			def        UpperCAmelCase__ (self			,   A			,   A			,   A = None			,   **A			,   ):
								return rescale(A			,   scale=A			,   data_format=A			,   **A      )

			def        UpperCAmelCase__ (self			,   A			,   A			,   A			,   A = None			,   **A			,   ):
								return normalize(A			,   mean=A			,   std=A			,   data_format=A			,   **A      )






			def        UpperCAmelCase__ (self			,   A			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = None			,   A = ChannelDimension.FIRST			,   **A			,   ):
								lowerCamelCase_					:      Optional[int]     =		do_resize if do_resize is not None else self.do_resize
								lowerCamelCase_					:      str     =		resample if resample is not None else self.resample
								lowerCamelCase_					:      Optional[Any]     =		do_rescale if do_rescale is not None else self.do_rescale
								lowerCamelCase_					:      Optional[Any]     =		rescale_factor if rescale_factor is not None else self.rescale_factor
								lowerCamelCase_					:      Optional[int]     =		do_normalize if do_normalize is not None else self.do_normalize
								lowerCamelCase_					:      Optional[int]     =		image_mean if image_mean is not None else self.image_mean
								lowerCamelCase_					:      List[Any]     =		image_std if image_std is not None else self.image_std
								lowerCamelCase_					:      Optional[Any]     =		do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

								lowerCamelCase_					:      int     =		size if size is not None else self.size
								lowerCamelCase_					:      Dict     =		get_size_dict(A			,   default_to_square=A      )

								lowerCamelCase_					:      Union[str, Any]     =		make_list_of_images(A      )

								if not valid_images(A      ):
													raise ValueError(
													    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
													    '''torch.Tensor, tf.Tensor or jax.ndarray.'''      )

								if do_resize and size is None or resample is None:
													raise ValueError('''Size and resample must be specified if do_resize is True.'''      )

								if do_rescale and rescale_factor is None:
													raise ValueError('''Rescale factor must be specified if do_rescale is True.'''      )

								if do_normalize and (image_mean is None or image_std is None):
													raise ValueError('''Image mean and std must be specified if do_normalize is True.'''      )

								# PIL RGBA images are converted to RGB
								if do_convert_rgb:
													lowerCamelCase_					:      Optional[int]     =		[convert_to_rgb(A      ) for image in images]

								# All transformations expect numpy arrays.
								lowerCamelCase_					:      str     =		[to_numpy_array(A      ) for image in images]

								if do_resize:
													lowerCamelCase_					:      List[str]     =		[self.resize(image=A			,   size=A			,   resample=A      ) for image in images]

								if do_rescale:
													lowerCamelCase_					:      List[str]     =		[self.rescale(image=A			,   scale=A      ) for image in images]

								if do_normalize:
													lowerCamelCase_					:      List[str]     =		[self.normalize(image=A			,   mean=A			,   std=A      ) for image in images]

								lowerCamelCase_					:      Optional[int]     =		[to_channel_dimension_format(A			,   A      ) for image in images]

								lowerCamelCase_					:      Union[str, Any]     =		BatchFeature(data={'''pixel_values''': images}			,   tensor_type=A      )

								return encoded_outputs




'''simple docstring'''


import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
__lowercase							:   Tuple					=     logging.getLogger()

__lowercase							:   str					=     logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)

class 	__lowercase				(  _lowercase    ):

			def        UpperCAmelCase__ (self			,   A      ):
								os.makedirs(A			,   exist_ok=A      )
								lowerCamelCase_					:      Any     =		{'''source''': '''What is love ?''', '''target''': '''life'''}
								lowerCamelCase_					:      List[str]     =		{'''train''': 1_2, '''val''': 2, '''test''': 2}
								for split in ["train", "test", "val"]:
													for field in ["source", "target"]:
																		lowerCamelCase_					:      List[Any]     =		'''\n'''.join([contents[field]] * n_lines[split]      )
																		with open(os.path.join(A			,   F"""{split}.{field}"""      )			,   '''w'''      ) as f:
																							f.write(A      )

			def        UpperCAmelCase__ (self			,   A			,   A = "pytorch"      ):
								lowerCamelCase_					:      List[str]     =		self.get_auto_remove_tmp_dir()
								lowerCamelCase_					:      Tuple     =		os.path.join(A			,   '''output'''      )
								lowerCamelCase_					:      Union[str, Any]     =		os.path.join(A			,   '''data'''      )
								self._create_dummy_data(data_dir=A      )

								lowerCamelCase_					:      Tuple     =		F"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

								if gpus > 0:
													testargs.append(F"""--gpus={gpus}"""      )
													if is_apex_available():
																		testargs.append('''--fp16'''      )
								else:
													testargs.append('''--gpus=0'''      )
													testargs.append('''--distributed_backend=ddp_cpu'''      )
													testargs.append('''--num_processes=2'''      )

								lowerCamelCase_					:      str     =		[sys.executable, str(Path(finetune_rag.__file__      ).resolve()      )] + testargs
								execute_subprocess_async(A			,   env=self.get_env()      )

								lowerCamelCase_					:      List[Any]     =		os.path.join(A			,   '''metrics.json'''      )
								with open(A      ) as f:
													lowerCamelCase_					:      List[str]     =		json.load(A      )
								return result

			@require_torch_gpu
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Tuple     =		self._run_finetune(gpus=1      )
								self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']			,   0.2      )

			@require_torch_multi_gpu
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      List[str]     =		self._run_finetune(gpus=2      )
								self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']			,   0.2      )

			@require_torch_gpu
			@require_ray
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		self._run_finetune(gpus=1			,   distributed_retriever='''ray'''      )
								self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']			,   0.2      )






			@require_torch_multi_gpu
			@require_ray
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Union[str, Any]     =		self._run_finetune(gpus=1			,   distributed_retriever='''ray'''      )
								self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']			,   0.2      )


