

"""simple docstring"""






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class      _UpperCAmelCase      (		datasets.BeamBasedBuilder):
			def 				__snake_case			(		self  )     ->     str:






				'''simple docstring'''







				return datasets.DatasetInfo(
				    features=datasets.Features({"""content""": datasets.Value("""string"""  )}  )   ,		supervised_keys=__lowerCAmelCase   ,		)
			def 				__snake_case			(		self   ,		_A   ,		_A  )     ->     Optional[int]:






				'''simple docstring'''







				return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,		gen_kwargs={"""examples""": get_test_dummy_examples()}  )]
			def 				__snake_case			(		self   ,		_A   ,		_A  )     ->     Union[str, Any]:






				'''simple docstring'''







				import apache_beam as beam

				return pipeline | "Load Examples" >> beam.Create(__lowerCAmelCase  )






class      _UpperCAmelCase      (		datasets.BeamBasedBuilder):
			def 				__snake_case			(		self  )     ->     List[str]:






				'''simple docstring'''







				return datasets.DatasetInfo(
				    features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""  )}  )}  )   ,		supervised_keys=__lowerCAmelCase   ,		)
			def 				__snake_case			(		self   ,		_A   ,		_A  )     ->     Any:






				'''simple docstring'''







				return [
				    datasets.SplitGenerator(name=datasets.Split.TRAIN   ,		gen_kwargs={"""examples""": get_test_nested_examples()}  )
				]
			def 				__snake_case			(		self   ,		_A   ,		_A  )     ->     Optional[Any]:






				'''simple docstring'''







				import apache_beam as beam

				return pipeline | "Load Examples" >> beam.Create(__lowerCAmelCase  )
def    UpperCamelCase       (      )					->      Optional[Any]:
	return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]			)]
def    UpperCamelCase       (      )					->      int:
	return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]			)]






class      _UpperCAmelCase      (		__a):
			@require_beam
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				_UpperCAmelCase     :				Any          =       len(get_test_dummy_examples()  )
				with tempfile.TemporaryDirectory() as tmp_cache_dir:
					_UpperCAmelCase     :				Tuple          =       DummyBeamDataset(cache_dir=__lowerCAmelCase   ,		beam_runner="""DirectRunner"""  )
					builder.download_and_prepare()
					self.assertTrue(
					    os.path.exists(
					        os.path.join(__lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		f'''{builder.name}-train.arrow'''  )  )  )
					self.assertDictEqual(builder.info.features   ,		datasets.Features({"""content""": datasets.Value("""string"""  )}  )  )
					_UpperCAmelCase     :				Optional[int]          =       builder.as_dataset()
					self.assertEqual(dset["""train"""].num_rows   ,		__lowerCAmelCase  )
					self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,		__lowerCAmelCase  )
					self.assertDictEqual(dset["""train"""][0]   ,		get_test_dummy_examples()[0][1]  )
					self.assertDictEqual(
					    dset["""train"""][expected_num_examples - 1]   ,		get_test_dummy_examples()[expected_num_examples - 1][1]  )
					self.assertTrue(
					    os.path.exists(os.path.join(__lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		"""dataset_info.json"""  )  )  )
					del dset
			@require_beam
			def 				__snake_case			(		self  )     ->     Dict:






				'''simple docstring'''







				import apache_beam as beam

				_UpperCAmelCase     :				Tuple          =       beam.io.parquetio.WriteToParquet

				_UpperCAmelCase     :				Optional[int]          =       len(get_test_dummy_examples()  )
				with tempfile.TemporaryDirectory() as tmp_cache_dir:
					_UpperCAmelCase     :				str          =       DummyBeamDataset(cache_dir=__lowerCAmelCase   ,		beam_runner="""DirectRunner"""  )
					with patch("""apache_beam.io.parquetio.WriteToParquet"""  ) as write_parquet_mock:
						_UpperCAmelCase     :				Optional[Any]          =       partial(__lowerCAmelCase   ,		num_shards=2  )
						builder.download_and_prepare()
					self.assertTrue(
					    os.path.exists(
					        os.path.join(
					            __lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		f'''{builder.name}-train-00000-of-00002.arrow'''  )  )  )
					self.assertTrue(
					    os.path.exists(
					        os.path.join(
					            __lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		f'''{builder.name}-train-00000-of-00002.arrow'''  )  )  )
					self.assertDictEqual(builder.info.features   ,		datasets.Features({"""content""": datasets.Value("""string"""  )}  )  )
					_UpperCAmelCase     :				str          =       builder.as_dataset()
					self.assertEqual(dset["""train"""].num_rows   ,		__lowerCAmelCase  )
					self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,		__lowerCAmelCase  )
					# Order is not preserved when sharding, so we just check that all the elements are there
					self.assertListEqual(sorted(dset["""train"""]["""content"""]  )   ,		sorted(["""foo""", """bar""", """foobar"""]  )  )
					self.assertTrue(
					    os.path.exists(os.path.join(__lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		"""dataset_info.json"""  )  )  )
					del dset
			@require_beam
			def 				__snake_case			(		self  )     ->     Tuple:






				'''simple docstring'''







				with tempfile.TemporaryDirectory() as tmp_cache_dir:
					_UpperCAmelCase     :				Optional[Any]          =       DummyBeamDataset(cache_dir=__lowerCAmelCase  )
					self.assertRaises(datasets.builder.MissingBeamOptions   ,		builder.download_and_prepare  )
			@require_beam
			def 				__snake_case			(		self  )     ->     int:






				'''simple docstring'''







				_UpperCAmelCase     :				Optional[Any]          =       len(get_test_nested_examples()  )
				with tempfile.TemporaryDirectory() as tmp_cache_dir:
					_UpperCAmelCase     :				List[Any]          =       NestedBeamDataset(cache_dir=__lowerCAmelCase   ,		beam_runner="""DirectRunner"""  )
					builder.download_and_prepare()
					self.assertTrue(
					    os.path.exists(
					        os.path.join(__lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		f'''{builder.name}-train.arrow'''  )  )  )
					self.assertDictEqual(
					    builder.info.features   ,		datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""  )}  )}  )  )
					_UpperCAmelCase     :				List[str]          =       builder.as_dataset()
					self.assertEqual(dset["""train"""].num_rows   ,		__lowerCAmelCase  )
					self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples   ,		__lowerCAmelCase  )
					self.assertDictEqual(dset["""train"""][0]   ,		get_test_nested_examples()[0][1]  )
					self.assertDictEqual(
					    dset["""train"""][expected_num_examples - 1]   ,		get_test_nested_examples()[expected_num_examples - 1][1]  )
					self.assertTrue(
					    os.path.exists(os.path.join(__lowerCAmelCase   ,		builder.name   ,		"""default"""   ,		"""0.0.0"""   ,		"""dataset_info.json"""  )  )  )
					del dset












import numpy as np







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return 1 / (1 + np.exp(-vector       ))







def   __lowerCamelCase   (				__a  :np.array       )						->							np.array:






		"""simple docstring"""

		return vector * sigmoid(1.702 * vector       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()



