

"""simple docstring"""




import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode
def    __SCREAMING_SNAKE_CASE						(						lowercase__			,							**lowercase__      ):
					"""simple docstring"""
					return RunBeamCommand(
					    args.dataset			,							args.name			,							args.cache_dir			,							args.beam_pipeline_options			,							args.data_dir			,							args.all_configs			,							args.save_info or args.save_infos			,							args.ignore_verifications			,							args.force_redownload			,							**lowercase__			,							)





class 			__UpperCamelCase (   _A						):


		@staticmethod
		def 					SCREAMING_SNAKE_CASE__   (__SCREAMING_SNAKE_CASE :		ArgumentParser):
							A			=					parser.add_parser("run_beam"     ,     help="Run a Beam dataset processing pipeline")
							run_beam_parser.add_argument("dataset"     ,     type=__SCREAMING_SNAKE_CASE     ,     help="Name of the dataset to download")
							run_beam_parser.add_argument("--name"     ,     type=__SCREAMING_SNAKE_CASE     ,     default=__SCREAMING_SNAKE_CASE     ,     help="Dataset config name")
							run_beam_parser.add_argument(
							    "--cache_dir"     ,     type=__SCREAMING_SNAKE_CASE     ,     default=__SCREAMING_SNAKE_CASE     ,     help="Cache directory where the datasets are stored"     ,     )
							run_beam_parser.add_argument(
							    "--beam_pipeline_options"     ,     type=__SCREAMING_SNAKE_CASE     ,     default=""     ,     help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"     ,     )
							run_beam_parser.add_argument(
							    "--data_dir"     ,     type=__SCREAMING_SNAKE_CASE     ,     default=__SCREAMING_SNAKE_CASE     ,     help="Can be used to specify a manual directory to get the files from"     ,     )
							run_beam_parser.add_argument("--all_configs"     ,     action="store_true"     ,     help="Test all dataset configurations")
							run_beam_parser.add_argument("--save_info"     ,     action="store_true"     ,     help="Save the dataset infos file")
							run_beam_parser.add_argument(
							    "--ignore_verifications"     ,     action="store_true"     ,     help="Run the test without checksums and splits checks")
							run_beam_parser.add_argument("--force_redownload"     ,     action="store_true"     ,     help="Force dataset redownload")
							# aliases
							run_beam_parser.add_argument("--save_infos"     ,     action="store_true"     ,     help="alias for save_info")
							run_beam_parser.set_defaults(func=__SCREAMING_SNAKE_CASE)


		def __init__(self :		int     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		str     ,     __SCREAMING_SNAKE_CASE :		bool     ,     __SCREAMING_SNAKE_CASE :		bool     ,     __SCREAMING_SNAKE_CASE :		bool     ,     __SCREAMING_SNAKE_CASE :		bool     ,     **__SCREAMING_SNAKE_CASE :		Optional[Any]     ,     ):
							A			=					dataset
							A			=					name
							A			=					cache_dir
							A			=					beam_pipeline_options
							A			=					data_dir
							A			=					all_configs
							A			=					save_infos
							A			=					ignore_verifications
							A			=					force_redownload
							A			=					config_kwargs


		def 					SCREAMING_SNAKE_CASE__   (self :		Dict):
							import apache_beam as beam

							if self._name is not None and self._all_configs:
												print("Both parameters `name` and `all_configs` can't be used at once.")
												exit(1)
							A			,    A			=					self._dataset, self._name
							A			=					dataset_module_factory(__SCREAMING_SNAKE_CASE)
							A			=					import_main_class(dataset_module.module_path)
							A			=					[]
							if self._beam_pipeline_options:
												A			=					beam.options.pipeline_options.PipelineOptions(
												    flags=[F"""--{opt.strip()}""" for opt in self._beam_pipeline_options.split(",") if opt])
							else:
												A			=					None
							if self._all_configs and len(builder_cls.BUILDER_CONFIGS) > 0:
												for builder_config in builder_cls.BUILDER_CONFIGS:
																	builders.append(
																	    builder_cls(
																	        config_name=builder_config.name     ,     data_dir=self._data_dir     ,     hash=dataset_module.hash     ,     beam_options=__SCREAMING_SNAKE_CASE     ,     cache_dir=self._cache_dir     ,     base_path=dataset_module.builder_kwargs.get("base_path")     ,     ))
							else:
												builders.append(
												    builder_cls(
												        config_name=__SCREAMING_SNAKE_CASE     ,     data_dir=self._data_dir     ,     beam_options=__SCREAMING_SNAKE_CASE     ,     cache_dir=self._cache_dir     ,     base_path=dataset_module.builder_kwargs.get("base_path")     ,     **self._config_kwargs     ,     ))

							for builder in builders:
												builder.download_and_prepare(
												    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
												    if not self._force_redownload
												    else DownloadMode.FORCE_REDOWNLOAD     ,     download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH)     ,     verification_mode=VerificationMode.NO_CHECKS
												    if self._ignore_verifications
												    else VerificationMode.ALL_CHECKS     ,     try_from_hf_gcs=__SCREAMING_SNAKE_CASE     ,     )
												if self._save_infos:
																	builder._save_infos()

							print("Apache beam run successful.")

							# If save_infos=True, the dataset infos file is created next to the loaded module file.
							# Let's move it to the original directory of the dataset script, to allow the user to
							# upload them on S3 at the same time afterwards.
							if self._save_infos:
												A			=					os.path.join(builder_cls.get_imported_module_dir()     ,     config.DATASETDICT_INFOS_FILENAME)

												A			=					Path(__SCREAMING_SNAKE_CASE).name + ".py"

												A			=					os.path.join(__SCREAMING_SNAKE_CASE     ,     __SCREAMING_SNAKE_CASE)
												if os.path.isfile(__SCREAMING_SNAKE_CASE):
																	A			=					os.path.dirname(__SCREAMING_SNAKE_CASE)
												elif os.path.isfile(__SCREAMING_SNAKE_CASE):
																	A			=					path
												else:  # in case of a remote dataset
																	print(F"""Dataset Infos file saved at {dataset_infos_path}""")
																	exit(1)

												# Move datasetinfo back to the user
												A			=					os.path.join(__SCREAMING_SNAKE_CASE     ,     config.DATASETDICT_INFOS_FILENAME)
												copyfile(__SCREAMING_SNAKE_CASE     ,     __SCREAMING_SNAKE_CASE)
												print(F"""Dataset Infos file saved at {user_dataset_infos_path}""")










'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


UpperCAmelCase			=    logging.get_logger(__name__)

UpperCAmelCase			=    {
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}

class __snake_case(      _lowerCAmelCase					):


						'''simple docstring'''



						UpperCAmelCase							:    Union[str, Any]           =		"bigbird_pegasus"
						UpperCAmelCase							:    Optional[Any]           =		["past_key_values"]
						UpperCAmelCase							:    Optional[Any]           =		{
						    "num_attention_heads": "encoder_attention_heads",
						    "hidden_size": "d_model",
						    "attention_probs_dropout_prob": "attention_dropout",
						}


						def __init__(					self	,       A_=9_6103	,       A_=4096	,       A_=16	,       A_=4096	,       A_=16	,       A_=16	,       A_=4096	,       A_=16	,       A_=0.0	,       A_=0.0	,       A_=True	,       A_=True	,       A_="gelu_new"	,       A_=1024	,       A_=0.1	,       A_=0.0	,       A_=0.0	,       A_=0.0_2	,       A_=2	,       A_=0.0	,       A_=True	,       A_=0	,       A_=2	,       A_=1	,       A_="block_sparse"	,       A_=64	,       A_=3	,       A_=False	,       **A_	,       )	->	List[str]:
							lowerCAmelCase    =	vocab_size
							lowerCAmelCase    =	max_position_embeddings
							lowerCAmelCase    =	d_model
							lowerCAmelCase    =	encoder_ffn_dim
							lowerCAmelCase    =	encoder_layers
							lowerCAmelCase    =	encoder_attention_heads
							lowerCAmelCase    =	decoder_ffn_dim
							lowerCAmelCase    =	decoder_layers
							lowerCAmelCase    =	decoder_attention_heads
							lowerCAmelCase    =	dropout
							lowerCAmelCase    =	attention_dropout
							lowerCAmelCase    =	activation_dropout
							lowerCAmelCase    =	activation_function
							lowerCAmelCase    =	init_std
							lowerCAmelCase    =	encoder_layerdrop
							lowerCAmelCase    =	decoder_layerdrop
							lowerCAmelCase    =	classifier_dropout
							lowerCAmelCase    =	use_cache
							lowerCAmelCase    =	encoder_layers
							lowerCAmelCase    =	scale_embedding  # scale factor will be sqrt(d_model) if True

							# extra config
							lowerCAmelCase    =	attention_type
							lowerCAmelCase    =	block_size
							lowerCAmelCase    =	num_random_blocks
							lowerCAmelCase    =	use_bias

							super().__init__(
							    pad_token_id=A_	,       bos_token_id=A_	,       eos_token_id=A_	,       is_encoder_decoder=A_	,       decoder_start_token_id=A_	,       **A_	,       )







class __snake_case(      _lowerCAmelCase					):


						'''simple docstring'''



						@property
						def 				__snake_case    (					self      )	->	Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								    ]      )

								if self.use_past:
									lowerCAmelCase    =	{0: """batch"""}
									lowerCAmelCase    =	{0: """batch""", 1: """past_decoder_sequence + sequence"""}
								else:
									lowerCAmelCase    =	{0: """batch""", 1: """decoder_sequence"""}
									lowerCAmelCase    =	{0: """batch""", 1: """decoder_sequence"""}

								if self.use_past:
									self.fill_with_past_key_values_(A_	,       direction="""inputs"""      )
							elif self.task == "causal-lm":
								# TODO: figure this case out.
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								    ]      )
								if self.use_past:
									lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
									for i in range(A_      ):
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
							else:
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
								        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
								    ]      )

							return common_inputs


						@property
						def 				__snake_case    (					self      )	->	Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	super().outputs
							else:
								lowerCAmelCase    =	super(A_	,       self      ).outputs
								if self.use_past:
									lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
									for i in range(A_      ):
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
							return common_outputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )

							# Generate decoder inputs
							lowerCAmelCase    =	seq_length if not self.use_past else 1
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )
							lowerCAmelCase    =	{f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
							lowerCAmelCase    =	dict(**A_	,       **A_      )

							if self.use_past:
								if not is_torch_available():
									raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
								else:
									import torch
								lowerCAmelCase,   lowerCAmelCase    =	common_inputs["""input_ids"""].shape
								lowerCAmelCase    =	common_inputs["""decoder_input_ids"""].shape[1]
								lowerCAmelCase,   lowerCAmelCase    =	self.num_attention_heads
								lowerCAmelCase    =	(
								    batch,
								    num_encoder_attention_heads,
								    encoder_seq_length,
								    self._config.hidden_size // num_encoder_attention_heads,
								)
								lowerCAmelCase    =	decoder_seq_length + 3
								lowerCAmelCase    =	(
								    batch,
								    num_decoder_attention_heads,
								    decoder_past_length,
								    self._config.hidden_size // num_decoder_attention_heads,
								)

								lowerCAmelCase    =	torch.cat(
								    [common_inputs["""decoder_attention_mask"""], torch.ones(A_	,       A_      )]	,       dim=1      )

								lowerCAmelCase    =	[]
								# If the number of encoder and decoder layers are present in the model configuration, both are considered
								lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
								lowerCAmelCase    =	min(A_	,       A_      )
								lowerCAmelCase    =	max(A_	,       A_      ) - min_num_layers
								lowerCAmelCase    =	"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

								for _ in range(A_      ):
									common_inputs["past_key_values"].append(
									    (
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									    )      )
								# TODO: test this.
								lowerCAmelCase    =	encoder_shape if remaining_side_name == """encoder""" else decoder_shape
								for _ in range(A_	,       A_      ):
									common_inputs["past_key_values"].append((torch.zeros(A_      ), torch.zeros(A_      ))      )
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )

							if self.use_past:
								if not is_torch_available():
									raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
								else:
									import torch
								lowerCAmelCase,   lowerCAmelCase    =	common_inputs["""input_ids"""].shape
								# Not using the same length for past_key_values
								lowerCAmelCase    =	seqlen + 2
								lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
								lowerCAmelCase,   lowerCAmelCase    =	self.num_attention_heads
								lowerCAmelCase    =	(
								    batch,
								    num_encoder_attention_heads,
								    past_key_values_length,
								    self._config.hidden_size // num_encoder_attention_heads,
								)

								lowerCAmelCase    =	common_inputs["""attention_mask"""].dtype
								lowerCAmelCase    =	torch.cat(
								    [common_inputs["""attention_mask"""], torch.ones(A_	,       A_	,       dtype=A_      )]	,       dim=1      )
								lowerCAmelCase    =	[
								    (torch.zeros(A_      ), torch.zeros(A_      )) for _ in range(A_      )
								]
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							# Copied from OnnxConfig.generate_dummy_inputs
							# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
							lowerCAmelCase    =	compute_effective_axis_dimension(
							    A_	,       fixed_dimension=OnnxConfig.default_fixed_batch	,       num_token_to_add=0      )

							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							lowerCAmelCase    =	tokenizer.num_special_tokens_to_add(A_      )
							lowerCAmelCase    =	compute_effective_axis_dimension(
							    A_	,       fixed_dimension=OnnxConfig.default_fixed_sequence	,       num_token_to_add=A_      )

							# Generate dummy inputs according to compute batch and sequence
							lowerCAmelCase    =	[""" """.join([tokenizer.unk_token]      ) * seq_length] * batch_size
							lowerCAmelCase    =	dict(tokenizer(A_	,       return_tensors=A_      )      )
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	self._generate_dummy_inputs_for_default_and_seqaseq_lm(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )

							elif self.task == "causal-lm":
								lowerCAmelCase    =	self._generate_dummy_inputs_for_causal_lm(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )
							else:
								lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )

							return common_inputs




						def 				__snake_case    (					self	,       A_	,       A_	,       A_	,       A_      )	->	Tuple:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	super()._flatten_past_key_values_(A_	,       A_	,       A_	,       A_      )
							else:
								lowerCAmelCase    =	super(A_	,       self      )._flatten_past_key_values_(
								    A_	,       A_	,       A_	,       A_      )