


"""simple docstring"""






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam





class 	lowerCamelCase__  (	datasets.BeamBasedBuilder							):







					"""simple docstring"""


					def lowerCamelCase__				(			self       :					int    ):
												'''simple docstring'''






												return datasets.DatasetInfo(
												    features=datasets.Features({"""content""": datasets.Value("""string"""    )}    )					,			supervised_keys=UpperCamelCase_					,			)






					def lowerCamelCase__				(			self       :					Any					,			UpperCamelCase       :					List[Any]					,			UpperCamelCase       :					Dict    ):
												'''simple docstring'''






												return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,			gen_kwargs={"""examples""": get_test_dummy_examples()}    )]






					def lowerCamelCase__				(			self       :					int					,			UpperCamelCase       :					List[str]					,			UpperCamelCase       :					Tuple    ):
												'''simple docstring'''






												import apache_beam as beam

												return pipeline | "Load Examples" >> beam.Create(UpperCamelCase_    )






class 	lowerCamelCase__  (	datasets.BeamBasedBuilder							):







					"""simple docstring"""


					def lowerCamelCase__				(			self       :					Any    ):
												'''simple docstring'''






												return datasets.DatasetInfo(
												    features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )					,			supervised_keys=UpperCamelCase_					,			)






					def lowerCamelCase__				(			self       :					Optional[int]					,			UpperCamelCase       :					Union[str, Any]					,			UpperCamelCase       :					Union[str, Any]    ):
												'''simple docstring'''






												return [
												    datasets.SplitGenerator(name=datasets.Split.TRAIN					,			gen_kwargs={"""examples""": get_test_nested_examples()}    )
												]






					def lowerCamelCase__				(			self       :					Optional[int]					,			UpperCamelCase       :					Optional[int]					,			UpperCamelCase       :					Tuple    ):
												'''simple docstring'''






												import apache_beam as beam

												return pipeline | "Load Examples" >> beam.Create(UpperCamelCase_    )






def 	lowerCamelCase			(	)						->			Optional[int]:

							'''simple docstring'''



							return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]				)]
def 	lowerCamelCase			(	)						->			Optional[Any]:

							'''simple docstring'''



							return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]				)]





class 	lowerCamelCase__  (	A							):







					"""simple docstring"""


					@require_beam
					def lowerCamelCase__				(			self       :					List[str]    ):
												'''simple docstring'''






												__UpperCAmelCase  :		Dict 				=   len(get_test_dummy_examples()    )
												with tempfile.TemporaryDirectory() as tmp_cache_dir:
																			__UpperCAmelCase  :		List[str] 				=   DummyBeamDataset(cache_dir=UpperCamelCase_					,			beam_runner="""DirectRunner"""    )
																			builder.download_and_prepare()
																			self.assertTrue(
																			    os.path.exists(
																			        os.path.join(UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			f'''{builder.name}-train.arrow'''    )    )    )
																			self.assertDictEqual(builder.info.features					,			datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
																			__UpperCAmelCase  :		int 				=   builder.as_dataset()
																			self.assertEqual(dset["""train"""].num_rows					,			UpperCamelCase_    )
																			self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					,			UpperCamelCase_    )
																			self.assertDictEqual(dset["""train"""][0]					,			get_test_dummy_examples()[0][1]    )
																			self.assertDictEqual(
																			    dset["""train"""][expected_num_examples - 1]					,			get_test_dummy_examples()[expected_num_examples - 1][1]    )
																			self.assertTrue(
																			    os.path.exists(os.path.join(UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			"""dataset_info.json"""    )    )    )
																			del dset






					@require_beam
					def lowerCamelCase__				(			self       :					Optional[int]    ):
												'''simple docstring'''






												import apache_beam as beam

												__UpperCAmelCase  :		Union[str, Any] 				=   beam.io.parquetio.WriteToParquet

												__UpperCAmelCase  :		List[Any] 				=   len(get_test_dummy_examples()    )
												with tempfile.TemporaryDirectory() as tmp_cache_dir:
																			__UpperCAmelCase  :		Optional[int] 				=   DummyBeamDataset(cache_dir=UpperCamelCase_					,			beam_runner="""DirectRunner"""    )
																			with patch("""apache_beam.io.parquetio.WriteToParquet"""    ) as write_parquet_mock:
																										__UpperCAmelCase  :		Optional[int] 				=   partial(UpperCamelCase_					,			num_shards=2    )
																										builder.download_and_prepare()
																			self.assertTrue(
																			    os.path.exists(
																			        os.path.join(
																			            UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			f'''{builder.name}-train-00000-of-00002.arrow'''    )    )    )
																			self.assertTrue(
																			    os.path.exists(
																			        os.path.join(
																			            UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			f'''{builder.name}-train-00000-of-00002.arrow'''    )    )    )
																			self.assertDictEqual(builder.info.features					,			datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
																			__UpperCAmelCase  :		Tuple 				=   builder.as_dataset()
																			self.assertEqual(dset["""train"""].num_rows					,			UpperCamelCase_    )
																			self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					,			UpperCamelCase_    )
																			# Order is not preserved when sharding, so we just check that all the elements are there
																			self.assertListEqual(sorted(dset["""train"""]["""content"""]    )					,			sorted(["""foo""", """bar""", """foobar"""]    )    )
																			self.assertTrue(
																			    os.path.exists(os.path.join(UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			"""dataset_info.json"""    )    )    )
																			del dset






					@require_beam
					def lowerCamelCase__				(			self       :					Optional[Any]    ):
												'''simple docstring'''






												with tempfile.TemporaryDirectory() as tmp_cache_dir:
																			__UpperCAmelCase  :		Any 				=   DummyBeamDataset(cache_dir=UpperCamelCase_    )
																			self.assertRaises(datasets.builder.MissingBeamOptions					,			builder.download_and_prepare    )






					@require_beam
					def lowerCamelCase__				(			self       :					Optional[int]    ):
												'''simple docstring'''






												__UpperCAmelCase  :		Optional[int] 				=   len(get_test_nested_examples()    )
												with tempfile.TemporaryDirectory() as tmp_cache_dir:
																			__UpperCAmelCase  :		Union[str, Any] 				=   NestedBeamDataset(cache_dir=UpperCamelCase_					,			beam_runner="""DirectRunner"""    )
																			builder.download_and_prepare()
																			self.assertTrue(
																			    os.path.exists(
																			        os.path.join(UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			f'''{builder.name}-train.arrow'''    )    )    )
																			self.assertDictEqual(
																			    builder.info.features					,			datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )    )
																			__UpperCAmelCase  :		str 				=   builder.as_dataset()
																			self.assertEqual(dset["""train"""].num_rows					,			UpperCamelCase_    )
																			self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					,			UpperCamelCase_    )
																			self.assertDictEqual(dset["""train"""][0]					,			get_test_nested_examples()[0][1]    )
																			self.assertDictEqual(
																			    dset["""train"""][expected_num_examples - 1]					,			get_test_nested_examples()[expected_num_examples - 1][1]    )
																			self.assertTrue(
																			    os.path.exists(os.path.join(UpperCamelCase_					,			builder.name					,			"""default"""					,			"""0.0.0"""					,			"""dataset_info.json"""    )    )    )
																			del dset







'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a__					:			Dict          =     {
    '''configuration_canine''': ['''CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CanineConfig'''],
    '''tokenization_canine''': ['''CanineTokenizer'''],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			a__					:			Union[str, Any]          =     [
			    '''CANINE_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''CanineForMultipleChoice''',
			    '''CanineForQuestionAnswering''',
			    '''CanineForSequenceClassification''',
			    '''CanineForTokenClassification''',
			    '''CanineLayer''',
			    '''CanineModel''',
			    '''CaninePreTrainedModel''',
			    '''load_tf_weights_in_canine''',
			]


if TYPE_CHECKING:
			from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
			from .tokenization_canine import CanineTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_canine import (
						    CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
						    CanineForMultipleChoice,
						    CanineForQuestionAnswering,
						    CanineForSequenceClassification,
						    CanineForTokenClassification,
						    CanineLayer,
						    CanineModel,
						    CaninePreTrainedModel,
						    load_tf_weights_in_canine,
						)


else:
			import sys

			a__					:			List[Any]          =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



