






'''simple docstring'''





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class  snake_case__      (					datasets.BeamBasedBuilder):




					def 		A      (  self  :		List[str]      )    ->							List[str]:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''content''': datasets.Value('''string'''      )}      )				, supervised_keys=_A				, )




					def 		A      (  self  :		Any				, _A  :		Any				, _A  :		int      )    ->							List[Any]:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''examples''': get_test_dummy_examples()}      )]





					def 		A      (  self  :		Any				, _A  :		int				, _A  :		List[Any]      )    ->							int:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(_A      )






class  snake_case__      (					datasets.BeamBasedBuilder):




					def 		A      (  self  :		Union[str, Any]      )    ->							Any:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''      )}      )}      )				, supervised_keys=_A				, )




					def 		A      (  self  :		List[str]				, _A  :		List[str]				, _A  :		Union[str, Any]      )    ->							Dict:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''examples''': get_test_nested_examples()}      )
								]





					def 		A      (  self  :		int				, _A  :		Optional[Any]				, _A  :		Tuple      )    ->							Dict:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(_A      )






def __UpperCAmelCase   (					)    ->   int:
			return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']		)]
def __UpperCAmelCase   (					)    ->   List[str]:
			return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']		)]







class  snake_case__      (					UpperCamelCase):




					@require_beam
					def 		A      (  self  :		Any      )    ->							int:
								UpperCAmelCase_      :       Dict          =       len(get_test_dummy_examples()      )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
											UpperCAmelCase_      :       List[str]          =       DummyBeamDataset(cache_dir=_A				, beam_runner='''DirectRunner'''      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(_A				, builder.name				, '''default'''				, '''0.0.0'''				, F"{builder.name}-train.arrow"      )      )      )
											self.assertDictEqual(builder.info.features				, datasets.Features({'''content''': datasets.Value('''string'''      )}      )      )
											UpperCAmelCase_      :       str          =       builder.as_dataset()
											self.assertEqual(dset['''train'''].num_rows				, _A      )
											self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples				, _A      )
											self.assertDictEqual(dset['''train'''][0]				, get_test_dummy_examples()[0][1]      )
											self.assertDictEqual(
											    dset['''train'''][expected_num_examples - 1]				, get_test_dummy_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(_A				, builder.name				, '''default'''				, '''0.0.0'''				, '''dataset_info.json'''      )      )      )
											del dset




					@require_beam
					def 		A      (  self  :		Optional[int]      )    ->							Any:
								import apache_beam as beam

								UpperCAmelCase_      :       Optional[Any]          =       beam.io.parquetio.WriteToParquet

								UpperCAmelCase_      :       int          =       len(get_test_dummy_examples()      )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
											UpperCAmelCase_      :       Optional[Any]          =       DummyBeamDataset(cache_dir=_A				, beam_runner='''DirectRunner'''      )
											with patch('''apache_beam.io.parquetio.WriteToParquet'''      ) as write_parquet_mock:
														UpperCAmelCase_      :       List[str]          =       partial(_A				, num_shards=2      )
														builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            _A				, builder.name				, '''default'''				, '''0.0.0'''				, F"{builder.name}-train-00000-of-00002.arrow"      )      )      )
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            _A				, builder.name				, '''default'''				, '''0.0.0'''				, F"{builder.name}-train-00000-of-00002.arrow"      )      )      )
											self.assertDictEqual(builder.info.features				, datasets.Features({'''content''': datasets.Value('''string'''      )}      )      )
											UpperCAmelCase_      :       List[str]          =       builder.as_dataset()
											self.assertEqual(dset['''train'''].num_rows				, _A      )
											self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples				, _A      )
											# Order is not preserved when sharding, so we just check that all the elements are there
											self.assertListEqual(sorted(dset['''train''']['''content''']      )				, sorted(['''foo''', '''bar''', '''foobar''']      )      )
											self.assertTrue(
											    os.path.exists(os.path.join(_A				, builder.name				, '''default'''				, '''0.0.0'''				, '''dataset_info.json'''      )      )      )
											del dset




					@require_beam
					def 		A      (  self  :		Dict      )    ->							Optional[Any]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
											UpperCAmelCase_      :       Optional[int]          =       DummyBeamDataset(cache_dir=_A      )
											self.assertRaises(datasets.builder.MissingBeamOptions				, builder.download_and_prepare      )





					@require_beam
					def 		A      (  self  :		str      )    ->							List[str]:
								UpperCAmelCase_      :       Dict          =       len(get_test_nested_examples()      )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
											UpperCAmelCase_      :       str          =       NestedBeamDataset(cache_dir=_A				, beam_runner='''DirectRunner'''      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(_A				, builder.name				, '''default'''				, '''0.0.0'''				, F"{builder.name}-train.arrow"      )      )      )
											self.assertDictEqual(
											    builder.info.features				, datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''      )}      )}      )      )
											UpperCAmelCase_      :       Dict          =       builder.as_dataset()
											self.assertEqual(dset['''train'''].num_rows				, _A      )
											self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples				, _A      )
											self.assertDictEqual(dset['''train'''][0]				, get_test_nested_examples()[0][1]      )
											self.assertDictEqual(
											    dset['''train'''][expected_num_examples - 1]				, get_test_nested_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(_A				, builder.name				, '''default'''				, '''0.0.0'''				, '''dataset_info.json'''      )      )      )
											del dset










'''simple docstring'''





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCamelCase						:						Optional[Any]  =					logging.get_logger(__name__)

_UpperCamelCase						:						Dict  =					{
    'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json',
}


class  snake_case__      (					UpperCamelCase):
					a_        =					"mvp"
					a_        =					["past_key_values"]
					a_        =					{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}




					def __init__(  self  :		Any				, _A  :		List[str]=5_02_67				, _A  :		Dict=10_24				, _A  :		List[str]=12				, _A  :		Dict=40_96				, _A  :		Optional[int]=16				, _A  :		Optional[Any]=12				, _A  :		int=40_96				, _A  :		Dict=16				, _A  :		Union[str, Any]=0.0				, _A  :		Optional[Any]=0.0				, _A  :		int="gelu"				, _A  :		Union[str, Any]=10_24				, _A  :		int=0.1				, _A  :		Optional[Any]=0.0				, _A  :		Optional[Any]=0.0				, _A  :		Optional[Any]=0.02				, _A  :		Optional[Any]=0.0				, _A  :		str=False				, _A  :		List[Any]=True				, _A  :		Optional[int]=1				, _A  :		Tuple=0				, _A  :		Dict=2				, _A  :		Dict=True				, _A  :		int=2				, _A  :		Tuple=2				, _A  :		List[str]=False				, _A  :		Any=1_00				, _A  :		Union[str, Any]=8_00				, **_A  :		List[str]				, )    ->							int:
								UpperCAmelCase_      :       Union[str, Any]          =       vocab_size
								UpperCAmelCase_      :       int          =       max_position_embeddings
								UpperCAmelCase_      :       Dict          =       d_model
								UpperCAmelCase_      :       Dict          =       encoder_ffn_dim
								UpperCAmelCase_      :       Optional[Any]          =       encoder_layers
								UpperCAmelCase_      :       Tuple          =       encoder_attention_heads
								UpperCAmelCase_      :       List[Any]          =       decoder_ffn_dim
								UpperCAmelCase_      :       List[Any]          =       decoder_layers
								UpperCAmelCase_      :       Dict          =       decoder_attention_heads
								UpperCAmelCase_      :       str          =       dropout
								UpperCAmelCase_      :       Optional[Any]          =       attention_dropout
								UpperCAmelCase_      :       Optional[Any]          =       activation_dropout
								UpperCAmelCase_      :       Union[str, Any]          =       activation_function
								UpperCAmelCase_      :       Union[str, Any]          =       init_std
								UpperCAmelCase_      :       Optional[int]          =       encoder_layerdrop
								UpperCAmelCase_      :       Optional[int]          =       decoder_layerdrop
								UpperCAmelCase_      :       Dict          =       classifier_dropout
								UpperCAmelCase_      :       List[Any]          =       use_cache
								UpperCAmelCase_      :       Dict          =       encoder_layers
								UpperCAmelCase_      :       Tuple          =       scale_embedding  # scale factor will be sqrt(d_model) if True
								UpperCAmelCase_      :       int          =       use_prompt
								UpperCAmelCase_      :       Tuple          =       prompt_length
								UpperCAmelCase_      :       str          =       prompt_mid_dim

								super().__init__(
								    pad_token_id=_A				, bos_token_id=_A				, eos_token_id=_A				, is_encoder_decoder=_A				, decoder_start_token_id=_A				, forced_eos_token_id=_A				, **_A				, )

								if self.forced_bos_token_id is None and kwargs.get('''force_bos_token_to_be_generated'''				, _A      ):
											UpperCAmelCase_      :       Optional[Any]          =       self.bos_token_id
											warnings.warn(
											    F"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. "
											    '''The config can simply be saved and uploaded again to be fixed.'''      )


