



"""simple docstring"""
import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class 			snake_case_						(						datasets.BeamBasedBuilder				):






							def snake_case_  (   self      ):
									return datasets.DatasetInfo(
									    features=datasets.Features({"content": datasets.Value("string"      )}      )			,     supervised_keys=a_			,     )






							def snake_case_  (   self			,     a_			,     a_      ):
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"examples": get_test_dummy_examples()}      )]






							def snake_case_  (   self			,     a_			,     a_      ):
									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(a_      )






class 			snake_case_						(						datasets.BeamBasedBuilder				):






							def snake_case_  (   self      ):
									return datasets.DatasetInfo(
									    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"      )}      )}      )			,     supervised_keys=a_			,     )






							def snake_case_  (   self			,     a_			,     a_      ):
									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"examples": get_test_nested_examples()}      )
									]






							def snake_case_  (   self			,     a_			,     a_      ):
									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(a_      )



def        lowerCAmelCase_						(							)			->      Any:
		return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]	)]



def        lowerCAmelCase_						(							)			->      Dict:
		return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]	)]






class 			snake_case_						(						a_				):






							@require_beam
							def snake_case_  (   self      ):
									a_     :      Any 						=     len(get_test_dummy_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Tuple 						=     DummyBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train.arrow"""      )      )      )
											self.assertDictEqual(builder.info.features			,     datasets.Features({"content": datasets.Value("string"      )}      )      )
											a_     :      Optional[Any] 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											self.assertDictEqual(dset["train"][0]			,     get_test_dummy_examples()[0][1]      )
											self.assertDictEqual(
											    dset["train"][expected_num_examples - 1]			,     get_test_dummy_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset






							@require_beam
							def snake_case_  (   self      ):
									import apache_beam as beam

									a_     :      List[Any] 						=     beam.io.parquetio.WriteToParquet

									a_     :      List[str] 						=     len(get_test_dummy_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Union[str, Any] 						=     DummyBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											with patch("apache_beam.io.parquetio.WriteToParquet"      ) as write_parquet_mock:
													a_     :      Optional[int] 						=     partial(a_			,     num_shards=2      )
													builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train-00000-of-00002.arrow"""      )      )      )
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train-00000-of-00002.arrow"""      )      )      )
											self.assertDictEqual(builder.info.features			,     datasets.Features({"content": datasets.Value("string"      )}      )      )
											a_     :      Dict 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											# Order is not preserved when sharding, so we just check that all the elements are there
											self.assertListEqual(sorted(dset["train"]["content"]      )			,     sorted(["foo", "bar", "foobar"]      )      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset






							@require_beam
							def snake_case_  (   self      ):
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Union[str, Any] 						=     DummyBeamDataset(cache_dir=a_      )
											self.assertRaises(datasets.builder.MissingBeamOptions			,     builder.download_and_prepare      )






							@require_beam
							def snake_case_  (   self      ):
									a_     :      Any 						=     len(get_test_nested_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Dict 						=     NestedBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train.arrow"""      )      )      )
											self.assertDictEqual(
											    builder.info.features			,     datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"      )}      )}      )      )
											a_     :      List[Any] 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											self.assertDictEqual(dset["train"][0]			,     get_test_nested_examples()[0][1]      )
											self.assertDictEqual(
											    dset["train"][expected_num_examples - 1]			,     get_test_nested_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset




"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_				= {
    """configuration_llama""": ["""LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LlamaConfig"""],
}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_				= ["""LlamaTokenizer"""]

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_				= ["""LlamaTokenizerFast"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					SCREAMING_SNAKE_CASE_				= [
					    """LlamaForCausalLM""",
					    """LlamaModel""",
					    """LlamaPreTrainedModel""",
					    """LlamaForSequenceClassification""",
					]


if TYPE_CHECKING:
					from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_llama import LlamaTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_llama_fast import LlamaTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
					import sys

					SCREAMING_SNAKE_CASE_				= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)