



"""simple docstring"""
import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam




class 			snake_case_						(						datasets.BeamBasedBuilder				):






							def snake_case_  (   self      ):
									return datasets.DatasetInfo(
									    features=datasets.Features({"content": datasets.Value("string"      )}      )			,     supervised_keys=a_			,     )






							def snake_case_  (   self			,     a_			,     a_      ):
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"examples": get_test_dummy_examples()}      )]






							def snake_case_  (   self			,     a_			,     a_      ):
									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(a_      )




class 			snake_case_						(						datasets.BeamBasedBuilder				):






							def snake_case_  (   self      ):
									return datasets.DatasetInfo(
									    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"      )}      )}      )			,     supervised_keys=a_			,     )






							def snake_case_  (   self			,     a_			,     a_      ):
									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN			,     gen_kwargs={"examples": get_test_nested_examples()}      )
									]






							def snake_case_  (   self			,     a_			,     a_      ):
									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(a_      )



def        lowerCAmelCase_						(							)			->      Any:
		return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]	)]



def        lowerCAmelCase_						(							)			->      Dict:
		return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]	)]




class 			snake_case_						(						a_				):






							@require_beam
							def snake_case_  (   self      ):
									a_     :      Any 						=     len(get_test_dummy_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Tuple 						=     DummyBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train.arrow"""      )      )      )
											self.assertDictEqual(builder.info.features			,     datasets.Features({"content": datasets.Value("string"      )}      )      )
											a_     :      Optional[Any] 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											self.assertDictEqual(dset["train"][0]			,     get_test_dummy_examples()[0][1]      )
											self.assertDictEqual(
											    dset["train"][expected_num_examples - 1]			,     get_test_dummy_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset






							@require_beam
							def snake_case_  (   self      ):
									import apache_beam as beam

									a_     :      List[Any] 						=     beam.io.parquetio.WriteToParquet

									a_     :      List[str] 						=     len(get_test_dummy_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Union[str, Any] 						=     DummyBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											with patch("apache_beam.io.parquetio.WriteToParquet"      ) as write_parquet_mock:
													a_     :      Optional[int] 						=     partial(a_			,     num_shards=2      )
													builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train-00000-of-00002.arrow"""      )      )      )
											self.assertTrue(
											    os.path.exists(
											        os.path.join(
											            a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train-00000-of-00002.arrow"""      )      )      )
											self.assertDictEqual(builder.info.features			,     datasets.Features({"content": datasets.Value("string"      )}      )      )
											a_     :      Dict 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											# Order is not preserved when sharding, so we just check that all the elements are there
											self.assertListEqual(sorted(dset["train"]["content"]      )			,     sorted(["foo", "bar", "foobar"]      )      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset






							@require_beam
							def snake_case_  (   self      ):
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Union[str, Any] 						=     DummyBeamDataset(cache_dir=a_      )
											self.assertRaises(datasets.builder.MissingBeamOptions			,     builder.download_and_prepare      )






							@require_beam
							def snake_case_  (   self      ):
									a_     :      Any 						=     len(get_test_nested_examples()      )
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
											a_     :      Dict 						=     NestedBeamDataset(cache_dir=a_			,     beam_runner="DirectRunner"      )
											builder.download_and_prepare()
											self.assertTrue(
											    os.path.exists(
											        os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     F"""{builder.name}-train.arrow"""      )      )      )
											self.assertDictEqual(
											    builder.info.features			,     datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"      )}      )}      )      )
											a_     :      List[Any] 						=     builder.as_dataset()
											self.assertEqual(dset["train"].num_rows			,     a_      )
											self.assertEqual(dset["train"].info.splits["train"].num_examples			,     a_      )
											self.assertDictEqual(dset["train"][0]			,     get_test_nested_examples()[0][1]      )
											self.assertDictEqual(
											    dset["train"][expected_num_examples - 1]			,     get_test_nested_examples()[expected_num_examples - 1][1]      )
											self.assertTrue(
											    os.path.exists(os.path.join(a_			,     builder.name			,     "default"			,     "0.0.0"			,     "dataset_info.json"      )      )      )
											del dset




"""simple docstring"""
import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType




class 			snake_case_						(						a_				):
							__lowerCAmelCase    					=					["image_processor", "tokenizer"]
							__lowerCAmelCase    					=					"ViltImageProcessor"
							__lowerCAmelCase    					=					("BertTokenizer", "BertTokenizerFast")






							def __init__(   self			,     a_=None			,     a_=None			,     **a_      ):
									a_     :      int 						=     None
									if "feature_extractor" in kwargs:
											warnings.warn(
											    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
											    " instead."			,     a_			,     )
											a_     :      List[Any] 						=     kwargs.pop("feature_extractor"      )

									a_     :      List[str] 						=     image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
											raise ValueError("You need to specify an `image_processor`."      )
									if tokenizer is None:
											raise ValueError("You need to specify a `tokenizer`."      )

									super().__init__(a_			,     a_      )
									a_     :      Dict 						=     self.image_processor






							def __call__(   self			,     a_			,     a_ = None			,     a_ = True			,     a_ = False			,     a_ = None			,     a_ = None			,     a_ = 0			,     a_ = None			,     a_ = None			,     a_ = None			,     a_ = False			,     a_ = False			,     a_ = False			,     a_ = False			,     a_ = True			,     a_ = None			,     **a_			,     ):
									a_     :      Union[str, Any] 						=     self.tokenizer(
									    text=a_			,     add_special_tokens=a_			,     padding=a_			,     truncation=a_			,     max_length=a_			,     stride=a_			,     pad_to_multiple_of=a_			,     return_token_type_ids=a_			,     return_attention_mask=a_			,     return_overflowing_tokens=a_			,     return_special_tokens_mask=a_			,     return_offsets_mapping=a_			,     return_length=a_			,     verbose=a_			,     return_tensors=a_			,     **a_			,     )
									# add pixel_values + pixel_mask
									a_     :      List[Any] 						=     self.image_processor(a_			,     return_tensors=a_      )
									encoding.update(a_      )

									return encoding






							def snake_case_  (   self			,     *a_			,     **a_      ):
									return self.tokenizer.batch_decode(*a_			,     **a_      )






							def snake_case_  (   self			,     *a_			,     **a_      ):
									return self.tokenizer.decode(*a_			,     **a_      )






							@property
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     self.tokenizer.model_input_names
									a_     :      Dict 						=     self.image_processor.model_input_names
									return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names      )      )






							@property
							def snake_case_  (   self      ):
									warnings.warn(
									    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."			,     a_			,     )
									return self.image_processor_class






							@property
							def snake_case_  (   self      ):
									warnings.warn(
									    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."			,     a_			,     )
									return self.image_processor