





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class 						UpperCamelCase_  (  datasets.BeamBasedBuilder							):



					def   __a     (		self     :     Optional[int]						):
							return datasets.DatasetInfo(
							    features=datasets.Features({'content': datasets.Value('string'						)}						)       ,      supervised_keys=lowerCamelCase       ,      )



					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     Optional[int]       ,      lowerCamelCase     :     Optional[int]						):
							return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,      gen_kwargs={'examples': get_test_dummy_examples()}						)]





					def   __a     (		self     :     str       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     List[Any]						):
							import apache_beam as beam

							return pipeline | "Load Examples" >> beam.Create(lowerCamelCase						)






class 						UpperCamelCase_  (  datasets.BeamBasedBuilder							):



					def   __a     (		self     :     Optional[int]						):
							return datasets.DatasetInfo(
							    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'						)}						)}						)       ,      supervised_keys=lowerCamelCase       ,      )



					def   __a     (		self     :     List[Any]       ,      lowerCamelCase     :     Optional[Any]       ,      lowerCamelCase     :     Tuple						):
							return [
							    datasets.SplitGenerator(name=datasets.Split.TRAIN       ,      gen_kwargs={'examples': get_test_nested_examples()}						)
							]





					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     int       ,      lowerCamelCase     :     List[str]						):
							import apache_beam as beam

							return pipeline | "Load Examples" >> beam.Create(lowerCamelCase						)


def 		_SCREAMING_SNAKE_CASE     ( ):
		return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']						)]


def 		_SCREAMING_SNAKE_CASE     ( ):
		return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']						)]






class 						UpperCamelCase_  (  snake_case__							):



					@require_beam
					def   __a     (		self     :     List[str]						):
							lowerCamelCase_	:     Tuple 						=      len(get_test_dummy_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
									lowerCamelCase_	:     Dict 						=      DummyBeamDataset(cache_dir=lowerCamelCase       ,      beam_runner='DirectRunner'						)
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      F"{builder.name}-train.arrow"						)						)						)
									self.assertDictEqual(builder.info.features       ,      datasets.Features({'content': datasets.Value('string'						)}						)						)
									lowerCamelCase_	:     Optional[Any] 						=      builder.as_dataset()
									self.assertEqual(dset['train'].num_rows       ,      lowerCamelCase						)
									self.assertEqual(dset['train'].info.splits['train'].num_examples       ,      lowerCamelCase						)
									self.assertDictEqual(dset['train'][0]       ,      get_test_dummy_examples()[0][1]						)
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]       ,      get_test_dummy_examples()[expected_num_examples - 1][1]						)
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      'dataset_info.json'						)						)						)
									del dset



					@require_beam
					def   __a     (		self     :     List[Any]						):
							import apache_beam as beam

							lowerCamelCase_	:     int 						=      beam.io.parquetio.WriteToParquet

							lowerCamelCase_	:     Any 						=      len(get_test_dummy_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
									lowerCamelCase_	:     List[str] 						=      DummyBeamDataset(cache_dir=lowerCamelCase       ,      beam_runner='DirectRunner'						)
									with patch('apache_beam.io.parquetio.WriteToParquet'						) as write_parquet_mock:
											lowerCamelCase_	:     Optional[Any] 						=      partial(lowerCamelCase       ,      num_shards=2						)
											builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      F"{builder.name}-train-00000-of-00002.arrow"						)						)						)
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      F"{builder.name}-train-00000-of-00002.arrow"						)						)						)
									self.assertDictEqual(builder.info.features       ,      datasets.Features({'content': datasets.Value('string'						)}						)						)
									lowerCamelCase_	:     List[str] 						=      builder.as_dataset()
									self.assertEqual(dset['train'].num_rows       ,      lowerCamelCase						)
									self.assertEqual(dset['train'].info.splits['train'].num_examples       ,      lowerCamelCase						)
									# Order is not preserved when sharding, so we just check that all the elements are there
									self.assertListEqual(sorted(dset['train']['content']						)       ,      sorted(['foo', 'bar', 'foobar']						)						)
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      'dataset_info.json'						)						)						)
									del dset



					@require_beam
					def   __a     (		self     :     Dict						):
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
									lowerCamelCase_	:     Optional[Any] 						=      DummyBeamDataset(cache_dir=lowerCamelCase						)
									self.assertRaises(datasets.builder.MissingBeamOptions       ,      builder.download_and_prepare						)





					@require_beam
					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     Dict 						=      len(get_test_nested_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
									lowerCamelCase_	:     List[str] 						=      NestedBeamDataset(cache_dir=lowerCamelCase       ,      beam_runner='DirectRunner'						)
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      F"{builder.name}-train.arrow"						)						)						)
									self.assertDictEqual(
									    builder.info.features       ,      datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'						)}						)}						)						)
									lowerCamelCase_	:     Optional[int] 						=      builder.as_dataset()
									self.assertEqual(dset['train'].num_rows       ,      lowerCamelCase						)
									self.assertEqual(dset['train'].info.splits['train'].num_examples       ,      lowerCamelCase						)
									self.assertDictEqual(dset['train'][0]       ,      get_test_nested_examples()[0][1]						)
									self.assertDictEqual(
									    dset['train'][expected_num_examples - 1]       ,      get_test_nested_examples()[expected_num_examples - 1][1]						)
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCamelCase       ,      builder.name       ,      'default'       ,      '0.0.0'       ,      'dataset_info.json'						)						)						)
									del dset







import os
import sys

_lowercase		:  List[str]			       =os.path.join(os.path.dirname(__file__), """src""")
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


_lowercase		:  Optional[int]			       =[
    """torch""",
    """numpy""",
    """tokenizers""",
    """filelock""",
    """requests""",
    """tqdm""",
    """regex""",
    """sentencepiece""",
    """sacremoses""",
    """importlib_metadata""",
    """huggingface_hub""",
]


@add_start_docstrings(AutoConfig.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoConfig.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)


@add_start_docstrings(AutoTokenizer.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoTokenizer.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)


@add_start_docstrings(AutoModel.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoModel.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)


@add_start_docstrings(AutoModelForCausalLM.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoModelForCausalLM.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)


@add_start_docstrings(AutoModelForMaskedLM.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoModelForMaskedLM.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)


@add_start_docstrings(AutoModelForSequenceClassification.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoModelForSequenceClassification.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)





@add_start_docstrings(AutoModelForQuestionAnswering.__doc__						)
def 		_SCREAMING_SNAKE_CASE     ( *lowerCAmelCase__							,**lowerCAmelCase__						):
		return AutoModelForQuestionAnswering.from_pretrained(*lowerCAmelCase__							,**lowerCAmelCase__						)
