



import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class       UpperCamelCase							(		datasets.BeamBasedBuilder      ):



	def 					__A				(  self						):
							return datasets.DatasetInfo(
							    features=datasets.Features({"content": datasets.Value("string"						)}						)					,			supervised_keys=__lowercase					,			)



	def 					__A				(  self					,			UpperCAmelCase__					,			UpperCAmelCase__						):
							return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,			gen_kwargs={"examples": get_test_dummy_examples()}						)]



	def 					__A				(  self					,			UpperCAmelCase__					,			UpperCAmelCase__						):
							import apache_beam as beam

							return pipeline | "Load Examples" >> beam.Create(__lowercase						)






class       UpperCamelCase							(		datasets.BeamBasedBuilder      ):



	def 					__A				(  self						):
							return datasets.DatasetInfo(
							    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"						)}						)}						)					,			supervised_keys=__lowercase					,			)



	def 					__A				(  self					,			UpperCAmelCase__					,			UpperCAmelCase__						):
							return [
							    datasets.SplitGenerator(name=datasets.Split.TRAIN					,			gen_kwargs={"examples": get_test_nested_examples()}						)
							]



	def 					__A				(  self					,			UpperCAmelCase__					,			UpperCAmelCase__						):
							import apache_beam as beam

							return pipeline | "Load Examples" >> beam.Create(__lowercase						)





def    UpperCamelCase		(  )->  Any:





						"""simple docstring"""





						return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]							)]
def    UpperCamelCase		(  )->  Optional[Any]:





						"""simple docstring"""





						return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]							)]






class       UpperCamelCase							(		snake_case__      ):



	@require_beam
	def 					__A				(  self						):
							A__					=  len(get_test_dummy_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
													A__					=  DummyBeamDataset(cache_dir=__lowercase					,			beam_runner="DirectRunner"						)
													builder.download_and_prepare()
													self.assertTrue(
													    os.path.exists(
													        os.path.join(__lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			F"""{builder.name}-train.arrow"""						)						)						)
													self.assertDictEqual(builder.info.features					,			datasets.Features({"content": datasets.Value("string"						)}						)						)
													A__					=  builder.as_dataset()
													self.assertEqual(dset["train"].num_rows					,			__lowercase						)
													self.assertEqual(dset["train"].info.splits["train"].num_examples					,			__lowercase						)
													self.assertDictEqual(dset["train"][0]					,			get_test_dummy_examples()[0][1]						)
													self.assertDictEqual(
													    dset["train"][expected_num_examples - 1]					,			get_test_dummy_examples()[expected_num_examples - 1][1]						)
													self.assertTrue(
													    os.path.exists(os.path.join(__lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			"dataset_info.json"						)						)						)
													del dset



	@require_beam
	def 					__A				(  self						):
							import apache_beam as beam

							A__					=  beam.io.parquetio.WriteToParquet

							A__					=  len(get_test_dummy_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
													A__					=  DummyBeamDataset(cache_dir=__lowercase					,			beam_runner="DirectRunner"						)
													with patch("apache_beam.io.parquetio.WriteToParquet"						) as write_parquet_mock:
																			A__					=  partial(__lowercase					,			num_shards=2						)
																			builder.download_and_prepare()
													self.assertTrue(
													    os.path.exists(
													        os.path.join(
													            __lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			F"""{builder.name}-train-00000-of-00002.arrow"""						)						)						)
													self.assertTrue(
													    os.path.exists(
													        os.path.join(
													            __lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			F"""{builder.name}-train-00000-of-00002.arrow"""						)						)						)
													self.assertDictEqual(builder.info.features					,			datasets.Features({"content": datasets.Value("string"						)}						)						)
													A__					=  builder.as_dataset()
													self.assertEqual(dset["train"].num_rows					,			__lowercase						)
													self.assertEqual(dset["train"].info.splits["train"].num_examples					,			__lowercase						)
													# Order is not preserved when sharding, so we just check that all the elements are there
													self.assertListEqual(sorted(dset["train"]["content"]						)					,			sorted(["foo", "bar", "foobar"]						)						)
													self.assertTrue(
													    os.path.exists(os.path.join(__lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			"dataset_info.json"						)						)						)
													del dset



	@require_beam
	def 					__A				(  self						):
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
													A__					=  DummyBeamDataset(cache_dir=__lowercase						)
													self.assertRaises(datasets.builder.MissingBeamOptions					,			builder.download_and_prepare						)



	@require_beam
	def 					__A				(  self						):
							A__					=  len(get_test_nested_examples()						)
							with tempfile.TemporaryDirectory() as tmp_cache_dir:
													A__					=  NestedBeamDataset(cache_dir=__lowercase					,			beam_runner="DirectRunner"						)
													builder.download_and_prepare()
													self.assertTrue(
													    os.path.exists(
													        os.path.join(__lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			F"""{builder.name}-train.arrow"""						)						)						)
													self.assertDictEqual(
													    builder.info.features					,			datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"						)}						)}						)						)
													A__					=  builder.as_dataset()
													self.assertEqual(dset["train"].num_rows					,			__lowercase						)
													self.assertEqual(dset["train"].info.splits["train"].num_examples					,			__lowercase						)
													self.assertDictEqual(dset["train"][0]					,			get_test_nested_examples()[0][1]						)
													self.assertDictEqual(
													    dset["train"][expected_num_examples - 1]					,			get_test_nested_examples()[expected_num_examples - 1][1]						)
													self.assertTrue(
													    os.path.exists(os.path.join(__lowercase					,			builder.name					,			"default"					,			"0.0.0"					,			"dataset_info.json"						)						)						)
													del dset












"""simple docstring"""





import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
							from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
							from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
							from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


_UpperCamelCase					      =   DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

_UpperCamelCase					      =   'main'
# Default branch name
_UpperCamelCase					      =   'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
_UpperCamelCase					      =   'aaaaaaa'
# This commit does not exist, so we should 404.

_UpperCamelCase					      =   'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
_UpperCamelCase					      =   '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Welcome!''' )
							yield
							print('''Bye!''' )



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Bonjour!''' )
							yield
							print('''Au revoir!''' )
class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				def        __lowercase	(  self	:str      ):
											# If the spec is missing, importlib would not be able to import the module dynamically.
											assert transformers.__spec__ is not None
											assert importlib.util.find_spec('''transformers'''      ) is not None

class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:str    ,     __lowercase	:str      ):
											with ContextManagers([]      ):
																		print('''Transformers are awesome!'''      )
											# The print statement adds a new line at the end of the output
											self.assertEqual(mock_stdout.getvalue()    ,     '''Transformers are awesome!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:Any    ,     __lowercase	:Tuple      ):
											with ContextManagers([context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Welcome!\nTransformers are awesome!\nBye!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:Optional[Any]      ):
											with ContextManagers([context_fr(), context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English and French welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''      )




				@require_torch
				def        __lowercase	(  self	:Tuple      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_tf
				def        __lowercase	(  self	:str      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_flax
				def        __lowercase	(  self	:str      ):
											# Flax models don't have labels
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     []      )




