


'''simple docstring'''






import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode


def 					_snake_case   (     A ,    **A       )						->    Optional[int]:
							return RunBeamCommand(
							    args.dataset ,    args.name ,    args.cache_dir ,    args.beam_pipeline_options ,    args.data_dir ,    args.all_configs ,    args.save_info or args.save_infos ,    args.ignore_verifications ,    args.force_redownload ,    **A ,    )






class 		a__				(							a__  ):





							'''simple docstring'''






							@staticmethod
							def   __SCREAMING_SNAKE_CASE							(		lowerCamelCase_   )       ->							List[str]:
														lowerCAmelCase__          =	parser.add_parser('''run_beam'''					,    help='''Run a Beam dataset processing pipeline'''   )
														run_beam_parser.add_argument('''dataset'''					,    type=lowerCamelCase_					,    help='''Name of the dataset to download'''   )
														run_beam_parser.add_argument('''--name'''					,    type=lowerCamelCase_					,    default=lowerCamelCase_					,    help='''Dataset config name'''   )
														run_beam_parser.add_argument(
														    '''--cache_dir'''					,    type=lowerCamelCase_					,    default=lowerCamelCase_					,    help='''Cache directory where the datasets are stored'''					,    )
														run_beam_parser.add_argument(
														    '''--beam_pipeline_options'''					,    type=lowerCamelCase_					,    default=''''''					,    help='''Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'''					,    )
														run_beam_parser.add_argument(
														    '''--data_dir'''					,    type=lowerCamelCase_					,    default=lowerCamelCase_					,    help='''Can be used to specify a manual directory to get the files from'''					,    )
														run_beam_parser.add_argument('''--all_configs'''					,    action='''store_true'''					,    help='''Test all dataset configurations'''   )
														run_beam_parser.add_argument('''--save_info'''					,    action='''store_true'''					,    help='''Save the dataset infos file'''   )
														run_beam_parser.add_argument(
														    '''--ignore_verifications'''					,    action='''store_true'''					,    help='''Run the test without checksums and splits checks'''   )
														run_beam_parser.add_argument('''--force_redownload'''					,    action='''store_true'''					,    help='''Force dataset redownload'''   )
														# aliases
														run_beam_parser.add_argument('''--save_infos'''					,    action='''store_true'''					,    help='''alias for save_info'''   )
														run_beam_parser.set_defaults(func=lowerCamelCase_   )







							def __init__(		self					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_					,    **lowerCamelCase_					,    )       ->							Union[str, Any]:
														lowerCAmelCase__          =	dataset
														lowerCAmelCase__          =	name
														lowerCAmelCase__          =	cache_dir
														lowerCAmelCase__          =	beam_pipeline_options
														lowerCAmelCase__          =	data_dir
														lowerCAmelCase__          =	all_configs
														lowerCAmelCase__          =	save_infos
														lowerCAmelCase__          =	ignore_verifications
														lowerCAmelCase__          =	force_redownload
														lowerCAmelCase__          =	config_kwargs







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							str:
														import apache_beam as beam

														if self._name is not None and self._all_configs:
																					print('''Both parameters `name` and `all_configs` can\'t be used at once.'''   )
																					exit(1   )
														lowerCAmelCase__   ,					lowerCAmelCase__          =	self._dataset, self._name
														lowerCAmelCase__          =	dataset_module_factory(lowerCamelCase_   )
														lowerCAmelCase__          =	import_main_class(dataset_module.module_path   )
														lowerCAmelCase__          =	[]
														if self._beam_pipeline_options:
																					lowerCAmelCase__          =	beam.options.pipeline_options.PipelineOptions(
																					    flags=[F"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(''','''   ) if opt]   )
														else:
																					lowerCAmelCase__          =	None
														if self._all_configs and len(builder_cls.BUILDER_CONFIGS   ) > 0:
																					for builder_config in builder_cls.BUILDER_CONFIGS:
																												builders.append(
																												    builder_cls(
																												        config_name=builder_config.name					,    data_dir=self._data_dir					,    hash=dataset_module.hash					,    beam_options=lowerCamelCase_					,    cache_dir=self._cache_dir					,    base_path=dataset_module.builder_kwargs.get('''base_path'''   )					,    )   )
														else:
																					builders.append(
																					    builder_cls(
																					        config_name=lowerCamelCase_					,    data_dir=self._data_dir					,    beam_options=lowerCamelCase_					,    cache_dir=self._cache_dir					,    base_path=dataset_module.builder_kwargs.get('''base_path'''   )					,    **self._config_kwargs					,    )   )

														for builder in builders:
																					builder.download_and_prepare(
																					    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
																					    if not self._force_redownload
																					    else DownloadMode.FORCE_REDOWNLOAD					,    download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH   )					,    verification_mode=VerificationMode.NO_CHECKS
																					    if self._ignore_verifications
																					    else VerificationMode.ALL_CHECKS					,    try_from_hf_gcs=lowerCamelCase_					,    )
																					if self._save_infos:
																												builder._save_infos()

														print('''Apache beam run successful.'''   )

														# If save_infos=True, the dataset infos file is created next to the loaded module file.
														# Let's move it to the original directory of the dataset script, to allow the user to
														# upload them on S3 at the same time afterwards.
														if self._save_infos:
																					lowerCAmelCase__          =	os.path.join(builder_cls.get_imported_module_dir()					,    config.DATASETDICT_INFOS_FILENAME   )

																					lowerCAmelCase__          =	Path(lowerCamelCase_   ).name + '''.py'''

																					lowerCAmelCase__          =	os.path.join(lowerCamelCase_					,    lowerCamelCase_   )
																					if os.path.isfile(lowerCamelCase_   ):
																												lowerCAmelCase__          =	os.path.dirname(lowerCamelCase_   )
																					elif os.path.isfile(lowerCamelCase_   ):
																												lowerCAmelCase__          =	path
																					else:  # in case of a remote dataset
																												print(F"""Dataset Infos file saved at {dataset_infos_path}"""   )
																												exit(1   )

																					# Move datasetinfo back to the user
																					lowerCAmelCase__          =	os.path.join(lowerCamelCase_					,    config.DATASETDICT_INFOS_FILENAME   )
																					copyfile(lowerCamelCase_					,    lowerCamelCase_   )
																					print(F"""Dataset Infos file saved at {user_dataset_infos_path}"""   )



'''simple docstring'''






import inspect
import unittest

from transformers import is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import MODEL_MAPPING, PoolFormerConfig, PoolFormerForImageClassification, PoolFormerModel
		from transformers.models.poolformer.modeling_poolformer import POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from PIL import Image

		from transformers import PoolFormerImageProcessor




class 		a__				(							a__  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Union[str, Any]:
														lowerCAmelCase__          =	self.config_class(**self.inputs_dict   )
														self.parent.assertTrue(hasattr(lowerCamelCase_					,    '''hidden_sizes'''   )   )
														self.parent.assertTrue(hasattr(lowerCamelCase_					,    '''num_encoder_blocks'''   )   )




class 		a__				:





							'''simple docstring'''






							def __init__(		self					,    lowerCamelCase_					,    lowerCamelCase_=13					,    lowerCamelCase_=64					,    lowerCamelCase_=3					,    lowerCamelCase_=4					,    lowerCamelCase_=[2, 2, 2, 2]					,    lowerCamelCase_=[8, 4, 2, 1]					,    lowerCamelCase_=[16, 32, 64, 1_28]					,    lowerCamelCase_=[1, 4, 8, 16]					,    lowerCamelCase_=False					,    lowerCamelCase_=True					,    lowerCamelCase_="gelu"					,    lowerCamelCase_=0.1					,    lowerCamelCase_=0.02					,    lowerCamelCase_=3					,    lowerCamelCase_=None					,    )       ->							Any:
														lowerCAmelCase__          =	parent
														lowerCAmelCase__          =	batch_size
														lowerCAmelCase__          =	image_size
														lowerCAmelCase__          =	num_channels
														lowerCAmelCase__          =	num_encoder_blocks
														lowerCAmelCase__          =	sr_ratios
														lowerCAmelCase__          =	depths
														lowerCAmelCase__          =	hidden_sizes
														lowerCAmelCase__          =	downsampling_rates
														lowerCAmelCase__          =	is_training
														lowerCAmelCase__          =	use_labels
														lowerCAmelCase__          =	hidden_act
														lowerCAmelCase__          =	hidden_dropout_prob
														lowerCAmelCase__          =	initializer_range
														lowerCAmelCase__          =	num_labels
														lowerCAmelCase__          =	scope







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Optional[int]:
														lowerCAmelCase__          =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

														lowerCAmelCase__          =	None
														if self.use_labels:
																					lowerCAmelCase__          =	ids_tensor([self.batch_size, self.image_size, self.image_size]					,    self.num_labels   )

														lowerCAmelCase__          =	PoolFormerConfig(
														    image_size=self.image_size					,    num_channels=self.num_channels					,    num_encoder_blocks=self.num_encoder_blocks					,    depths=self.depths					,    hidden_sizes=self.hidden_sizes					,    hidden_act=self.hidden_act					,    hidden_dropout_prob=self.hidden_dropout_prob					,    initializer_range=self.initializer_range					,    )

														return config, pixel_values, labels







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )       ->							Any:
														lowerCAmelCase__          =	PoolFormerModel(config=lowerCamelCase_   )
														model.to(lowerCamelCase_   )
														model.eval()
														lowerCAmelCase__          =	model(lowerCamelCase_   )
														lowerCAmelCase__          =	lowerCAmelCase__          =	self.image_size // 32.0
														self.parent.assertEqual(
														    result.last_hidden_state.shape					,    (self.batch_size, self.hidden_sizes[-1], expected_height, expected_width)   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							str:
														lowerCAmelCase__          =	self.prepare_config_and_inputs()
														lowerCAmelCase__   ,					lowerCAmelCase__   ,					lowerCAmelCase__          =	config_and_inputs
														lowerCAmelCase__          =	{'''pixel_values''': pixel_values}
														return config, inputs_dict




@require_torch
class 		a__				(							a__				,				a__				,				unittest.TestCase  ):





							'''simple docstring'''






							lowercase__			:   Dict							=      (PoolFormerModel, PoolFormerForImageClassification) if is_torch_available() else ()
							lowercase__			:   Any							=      (
							    {"feature-extraction": PoolFormerModel, "image-classification": PoolFormerForImageClassification}
							    if is_torch_available()
							    else {}
							)

							lowercase__			:   Optional[int]							=      False
							lowercase__			:   Optional[int]							=      False
							lowercase__			:   Union[str, Any]							=      False
							lowercase__			:   List[Any]							=      False
							lowercase__			:   Tuple							=      False







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Union[str, Any]:
														lowerCAmelCase__          =	PoolFormerModelTester(self   )
														lowerCAmelCase__          =	PoolFormerConfigTester(self					,    config_class=lowerCamelCase_   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							List[Any]:
														self.config_tester.run_common_tests()







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							List[Any]:
														lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*lowerCamelCase_   )







							@unittest.skip('''PoolFormer does not use inputs_embeds'''   )
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							str:
														pass







							@unittest.skip('''PoolFormer does not have get_input_embeddings method and get_output_embeddings methods'''   )
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Tuple:
														pass







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														def check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   ):
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					model.to(lowerCamelCase_   )
																					model.eval()

																					with torch.no_grad():
																												lowerCAmelCase__          =	model(**self._prepare_for_class(lowerCamelCase_					,    lowerCamelCase_   )   )

																					lowerCAmelCase__          =	outputs.hidden_states

																					lowerCAmelCase__          =	self.model_tester.num_encoder_blocks
																					self.assertEqual(len(lowerCamelCase_   )					,    lowerCamelCase_   )

																					# verify the first hidden states (first block)
																					self.assertListEqual(
																					    list(hidden_states[0].shape[-3:]   )					,    [
																					        self.model_tester.hidden_sizes[0],
																					        self.model_tester.image_size // 4,
																					        self.model_tester.image_size // 4,
																					    ]					,    )

														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__          =	True
																					check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )

																					# check that output_hidden_states also work using config
																					del inputs_dict["output_hidden_states"]
																					lowerCAmelCase__          =	True

																					check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														if not self.model_tester.is_training:
																					return

														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()
														lowerCAmelCase__          =	True

														for model_class in self.all_model_classes:
																					if model_class in get_values(lowerCamelCase_   ):
																												continue
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					model.to(lowerCamelCase_   )
																					model.train()
																					lowerCAmelCase__          =	self._prepare_for_class(lowerCamelCase_					,    lowerCamelCase_					,    return_labels=lowerCamelCase_   )
																					lowerCAmelCase__          =	model(**lowerCamelCase_   ).loss
																					loss.backward()







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					lowerCAmelCase__          =	inspect.signature(model.forward   )
																					# signature.parameters is an OrderedDict => so arg_names order is deterministic
																					lowerCAmelCase__          =	[*signature.parameters.keys()]

																					lowerCAmelCase__          =	['''pixel_values''']
																					self.assertListEqual(arg_names[:1]					,    lowerCamelCase_   )







							@slow
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														for model_name in POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					lowerCAmelCase__          =	PoolFormerModel.from_pretrained(lowerCamelCase_   )
																					self.assertIsNotNone(lowerCamelCase_   )


def 					_snake_case   (     )						->    Optional[int]:
							lowerCAmelCase__          =	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       )
							return image






@require_torch
class 		a__				(							unittest.TestCase  ):





							'''simple docstring'''






							@slow
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Dict:
														lowerCAmelCase__          =	PoolFormerImageProcessor()
														lowerCAmelCase__          =	PoolFormerForImageClassification.from_pretrained('''sail/poolformer_s12'''   ).to(lowerCamelCase_   )

														lowerCAmelCase__          =	image_processor(images=prepare_img()					,    return_tensors='''pt'''   ).to(lowerCamelCase_   )

														# forward pass
														with torch.no_grad():
																					lowerCAmelCase__          =	model(**lowerCamelCase_   )

														# verify the logits
														lowerCAmelCase__          =	torch.Size((1, 10_00)   )
														self.assertEqual(outputs.logits.shape					,    lowerCamelCase_   )

														lowerCAmelCase__          =	torch.tensor([-0.6_113, 0.1_685, -0.0_492]   ).to(lowerCamelCase_   )
														self.assertTrue(torch.allclose(outputs.logits[0, :3]					,    lowerCamelCase_					,    atol=1e-4   )   )