


"""simple docstring"""



import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam







class _lowerCAmelCase       (   datasets.BeamBasedBuilder			):







   def 				lowerCamelCase	(			self		)			->	Union[str, Any]:

       '''simple docstring'''




       return datasets.DatasetInfo(
           features=datasets.Features({"content": datasets.Value("string"		)}		)		,			supervised_keys=UpperCamelCase__		,			)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__		)			->	Dict:

       '''simple docstring'''




       return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={"examples": get_test_dummy_examples()}		)]







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__		)			->	Any:

       '''simple docstring'''




       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(UpperCamelCase__		)







class _lowerCAmelCase       (   datasets.BeamBasedBuilder			):







   def 				lowerCamelCase	(			self		)			->	Union[str, Any]:

       '''simple docstring'''




       return datasets.DatasetInfo(
           features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"		)}		)}		)		,			supervised_keys=UpperCamelCase__		,			)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__		)			->	Optional[int]:

       '''simple docstring'''




       return [
           datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={"examples": get_test_nested_examples()}		)
       ]







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__		)			->	str:

       '''simple docstring'''




       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(UpperCamelCase__		)







def 						__lowerCAmelCase		(		)			->		Dict:


    """simple docstring"""





    return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]				)]







def 						__lowerCAmelCase		(		)			->		Tuple:


    """simple docstring"""





    return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]				)]







class _lowerCAmelCase       (   snake_case_			):







   @require_beam
   def 				lowerCamelCase	(			self		)			->	Tuple:

       '''simple docstring'''




       snake_case					:   Tuple         =					len(get_test_dummy_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
           snake_case					:   Union[str, Any]         =					DummyBeamDataset(cache_dir=UpperCamelCase__		,			beam_runner="DirectRunner"		)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			F'{builder.name}-train.arrow'		)		)		)
           self.assertDictEqual(builder.info.features		,			datasets.Features({"content": datasets.Value("string"		)}		)		)
           snake_case					:   Dict         =					builder.as_dataset()
           self.assertEqual(dset["train"].num_rows		,			UpperCamelCase__		)
           self.assertEqual(dset["train"].info.splits["train"].num_examples		,			UpperCamelCase__		)
           self.assertDictEqual(dset["train"][0]		,			get_test_dummy_examples()[0][1]		)
           self.assertDictEqual(
               dset["train"][expected_num_examples - 1]		,			get_test_dummy_examples()[expected_num_examples - 1][1]		)
           self.assertTrue(
               os.path.exists(os.path.join(UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			"dataset_info.json"		)		)		)
           del dset







   @require_beam
   def 				lowerCamelCase	(			self		)			->	Union[str, Any]:

       '''simple docstring'''




       import apache_beam as beam

       snake_case					:   str         =					beam.io.parquetio.WriteToParquet

       snake_case					:   List[str]         =					len(get_test_dummy_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
           snake_case					:   List[str]         =					DummyBeamDataset(cache_dir=UpperCamelCase__		,			beam_runner="DirectRunner"		)
           with patch("apache_beam.io.parquetio.WriteToParquet"		) as write_parquet_mock:
               snake_case					:   int         =					partial(UpperCamelCase__		,			num_shards=2		)
               builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			F'{builder.name}-train-00000-of-00002.arrow'		)		)		)
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			F'{builder.name}-train-00000-of-00002.arrow'		)		)		)
           self.assertDictEqual(builder.info.features		,			datasets.Features({"content": datasets.Value("string"		)}		)		)
           snake_case					:   Optional[Any]         =					builder.as_dataset()
           self.assertEqual(dset["train"].num_rows		,			UpperCamelCase__		)
           self.assertEqual(dset["train"].info.splits["train"].num_examples		,			UpperCamelCase__		)
           # Order is not preserved when sharding, so we just check that all the elements are there
           self.assertListEqual(sorted(dset["train"]["content"]		)		,			sorted(["foo", "bar", "foobar"]		)		)
           self.assertTrue(
               os.path.exists(os.path.join(UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			"dataset_info.json"		)		)		)
           del dset







   @require_beam
   def 				lowerCamelCase	(			self		)			->	Union[str, Any]:

       '''simple docstring'''




       with tempfile.TemporaryDirectory() as tmp_cache_dir:
           snake_case					:   Tuple         =					DummyBeamDataset(cache_dir=UpperCamelCase__		)
           self.assertRaises(datasets.builder.MissingBeamOptions		,			builder.download_and_prepare		)







   @require_beam
   def 				lowerCamelCase	(			self		)			->	Union[str, Any]:

       '''simple docstring'''




       snake_case					:   Dict         =					len(get_test_nested_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
           snake_case					:   Union[str, Any]         =					NestedBeamDataset(cache_dir=UpperCamelCase__		,			beam_runner="DirectRunner"		)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			F'{builder.name}-train.arrow'		)		)		)
           self.assertDictEqual(
               builder.info.features		,			datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"		)}		)}		)		)
           snake_case					:   List[Any]         =					builder.as_dataset()
           self.assertEqual(dset["train"].num_rows		,			UpperCamelCase__		)
           self.assertEqual(dset["train"].info.splits["train"].num_examples		,			UpperCamelCase__		)
           self.assertDictEqual(dset["train"][0]		,			get_test_nested_examples()[0][1]		)
           self.assertDictEqual(
               dset["train"][expected_num_examples - 1]		,			get_test_nested_examples()[expected_num_examples - 1][1]		)
           self.assertTrue(
               os.path.exists(os.path.join(UpperCamelCase__		,			builder.name		,			"default"		,			"0.0.0"		,			"dataset_info.json"		)		)		)
           del dset









"""simple docstring"""



import baseaa







def 						__lowerCAmelCase		(		lowercase						:						str				)			->		bytes:


    """simple docstring"""





    return baseaa.aaaencode(string.encode("utf-8"				)				)







def 						__lowerCAmelCase		(		lowercase						:						bytes				)			->		str:


    """simple docstring"""





    return baseaa.aaadecode(lowercase				).decode("utf-8"				)


if __name__ == "__main__":
  import doctest

  doctest.testmod()





