



"""simple docstring"""







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class      UpperCAmelCase__		(      datasets.BeamBasedBuilder     ):



		def 					A_		( self  :    List[str]				)			->						Dict:






									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'content': datasets.Value('string'				)}				)      ,							supervised_keys=snake_case      ,							)



		def 					A_		( self  :    int      ,							snake_case  :    int      ,							snake_case  :    Optional[Any]				)			->						Union[str, Any]:






									'''simple docstring'''

									return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,							gen_kwargs={'examples': get_test_dummy_examples()}				)]



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    Optional[int]      ,							snake_case  :    Dict				)			->						Any:






									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(snake_case				)






class      UpperCAmelCase__		(      datasets.BeamBasedBuilder     ):



		def 					A_		( self  :    Tuple				)			->						Union[str, Any]:






									'''simple docstring'''

									return datasets.DatasetInfo(
									    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)      ,							supervised_keys=snake_case      ,							)



		def 					A_		( self  :    Optional[Any]      ,							snake_case  :    Dict      ,							snake_case  :    Tuple				)			->						int:






									'''simple docstring'''

									return [
									    datasets.SplitGenerator(name=datasets.Split.TRAIN      ,							gen_kwargs={'examples': get_test_nested_examples()}				)
									]



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    int      ,							snake_case  :    Union[str, Any]				)			->						Union[str, Any]:






									'''simple docstring'''

									import apache_beam as beam

									return pipeline | "Load Examples" >> beam.Create(snake_case				)



def   lowerCAmelCase__			(     ) ->							int:
							return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']			)]

def   lowerCAmelCase__			(     ) ->							List[str]:
							return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']			)]






class      UpperCAmelCase__		(      UpperCamelCase     ):



		@require_beam
		def 					A_		( self  :    List[Any]				)			->						int:






									'''simple docstring'''

									A         =					len(get_test_dummy_examples()				)
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
																A         =					DummyBeamDataset(cache_dir=snake_case      ,							beam_runner='DirectRunner'				)
																builder.download_and_prepare()
																self.assertTrue(
																    os.path.exists(
																        os.path.join(snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							f"""{builder.name}-train.arrow"""				)				)				)
																self.assertDictEqual(builder.info.features      ,							datasets.Features({'content': datasets.Value('string'				)}				)				)
																A         =					builder.as_dataset()
																self.assertEqual(dset['train'].num_rows      ,							snake_case				)
																self.assertEqual(dset['train'].info.splits['train'].num_examples      ,							snake_case				)
																self.assertDictEqual(dset['train'][0]      ,							get_test_dummy_examples()[0][1]				)
																self.assertDictEqual(
																    dset['train'][expected_num_examples - 1]      ,							get_test_dummy_examples()[expected_num_examples - 1][1]				)
																self.assertTrue(
																    os.path.exists(os.path.join(snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							'dataset_info.json'				)				)				)
																del dset



		@require_beam
		def 					A_		( self  :    Any				)			->						Union[str, Any]:






									'''simple docstring'''

									import apache_beam as beam

									A         =					beam.io.parquetio.WriteToParquet

									A         =					len(get_test_dummy_examples()				)
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
																A         =					DummyBeamDataset(cache_dir=snake_case      ,							beam_runner='DirectRunner'				)
																with patch('apache_beam.io.parquetio.WriteToParquet'				) as write_parquet_mock:
																							A         =					partial(snake_case      ,							num_shards=2				)
																							builder.download_and_prepare()
																self.assertTrue(
																    os.path.exists(
																        os.path.join(
																            snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							f"""{builder.name}-train-00000-of-00002.arrow"""				)				)				)
																self.assertTrue(
																    os.path.exists(
																        os.path.join(
																            snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							f"""{builder.name}-train-00000-of-00002.arrow"""				)				)				)
																self.assertDictEqual(builder.info.features      ,							datasets.Features({'content': datasets.Value('string'				)}				)				)
																A         =					builder.as_dataset()
																self.assertEqual(dset['train'].num_rows      ,							snake_case				)
																self.assertEqual(dset['train'].info.splits['train'].num_examples      ,							snake_case				)
																# Order is not preserved when sharding, so we just check that all the elements are there
																self.assertListEqual(sorted(dset['train']['content']				)      ,							sorted(['foo', 'bar', 'foobar']				)				)
																self.assertTrue(
																    os.path.exists(os.path.join(snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							'dataset_info.json'				)				)				)
																del dset



		@require_beam
		def 					A_		( self  :    Any				)			->						Optional[Any]:






									'''simple docstring'''

									with tempfile.TemporaryDirectory() as tmp_cache_dir:
																A         =					DummyBeamDataset(cache_dir=snake_case				)
																self.assertRaises(datasets.builder.MissingBeamOptions      ,							builder.download_and_prepare				)



		@require_beam
		def 					A_		( self  :    Tuple				)			->						Any:






									'''simple docstring'''

									A         =					len(get_test_nested_examples()				)
									with tempfile.TemporaryDirectory() as tmp_cache_dir:
																A         =					NestedBeamDataset(cache_dir=snake_case      ,							beam_runner='DirectRunner'				)
																builder.download_and_prepare()
																self.assertTrue(
																    os.path.exists(
																        os.path.join(snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							f"""{builder.name}-train.arrow"""				)				)				)
																self.assertDictEqual(
																    builder.info.features      ,							datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)				)
																A         =					builder.as_dataset()
																self.assertEqual(dset['train'].num_rows      ,							snake_case				)
																self.assertEqual(dset['train'].info.splits['train'].num_examples      ,							snake_case				)
																self.assertDictEqual(dset['train'][0]      ,							get_test_nested_examples()[0][1]				)
																self.assertDictEqual(
																    dset['train'][expected_num_examples - 1]      ,							get_test_nested_examples()[expected_num_examples - 1][1]				)
																self.assertTrue(
																    os.path.exists(os.path.join(snake_case      ,							builder.name      ,							'default'      ,							'0.0.0'      ,							'dataset_info.json'				)				)				)
																del dset








"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


A    =			logging.get_logger(__name__)

A    =			{
    'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/config.json',
    'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/config.json'
    # See all FNet models at https://huggingface.co/models?filter=fnet
}






class      UpperCAmelCase__		(      UpperCamelCase     ):
		lowerCAmelCase_       :	List[Any] 							=     """fnet"""



		def __init__( self  :    Tuple      ,							snake_case  :    Optional[int]=32_000      ,							snake_case  :    Optional[Any]=768      ,							snake_case  :    Union[str, Any]=12      ,							snake_case  :    List[str]=3_072      ,							snake_case  :    str="gelu_new"      ,							snake_case  :    Any=0.1      ,							snake_case  :    List[str]=512      ,							snake_case  :    Any=4      ,							snake_case  :    str=0.02      ,							snake_case  :    Union[str, Any]=1E-1_2      ,							snake_case  :    Optional[Any]=False      ,							snake_case  :    List[Any]=512      ,							snake_case  :    int=3      ,							snake_case  :    Any=1      ,							snake_case  :    Union[str, Any]=2      ,							**snake_case  :    Union[str, Any]      ,							)			->						Dict:






									'''simple docstring'''

									super().__init__(pad_token_id=snake_case      ,							bos_token_id=snake_case      ,							eos_token_id=snake_case      ,							**snake_case				)

									A         =					vocab_size
									A         =					max_position_embeddings
									A         =					hidden_size
									A         =					num_hidden_layers
									A         =					intermediate_size
									A         =					hidden_act
									A         =					hidden_dropout_prob
									A         =					initializer_range
									A         =					type_vocab_size
									A         =					layer_norm_eps
									A         =					use_tpu_fourier_optimizations
									A         =					tpu_short_seq_length



