





"""simple docstring"""





from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)


def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		List[Any]				,		SCREAMING_SNAKE_CASE	:		int     ):







      '''simple docstring'''




      lowerCAmelCase    		=			[]
      for part_id in partition_order:
            lowerCAmelCase    		=			df.where(F'SPARK_PARTITION_ID() = {part_id}'     ).collect()
            for row_idx, row in enumerate(SCREAMING_SNAKE_CASE     ):
                  expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())     )
      return expected_row_ids_and_row_dicts


@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(1_00     ).repartition(1     )
      lowerCAmelCase    		=			Spark(SCREAMING_SNAKE_CASE     )
      # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
      # that each partition can hold 2 rows.
      spark_builder._repartition_df_if_needed(max_shard_size=16     )
      # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
      assert spark_builder.df.rdd.getNumPartitions() == 50


@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(10     ).repartition(2     )
      lowerCAmelCase    		=			[1, 0]
      lowerCAmelCase    		=			_generate_iterable_examples(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     )  # Reverse the partitions.
      lowerCAmelCase    		=			_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     )

      for i, (row_id, row_dict) in enumerate(generate_fn()     ):
            lowerCAmelCase		,						lowerCAmelCase    		=			expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(10     ).repartition(1     )
      lowerCAmelCase    		=			SparkExamplesIterable(SCREAMING_SNAKE_CASE     )
      assert it.n_shards == 1
      for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE     ):
            assert row_id == F'0_{i}'
            assert row_dict == {"id": i}


@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(30     ).repartition(3     )
      # Mock the generator so that shuffle reverses the partition indices.
      with patch("""numpy.random.Generator"""     ) as generator_mock:
            lowerCAmelCase    		=			lambda SCREAMING_SNAKE_CASE     : x.reverse()
            lowerCAmelCase    		=			_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE				,		[2, 1, 0]     )

            lowerCAmelCase    		=			SparkExamplesIterable(SCREAMING_SNAKE_CASE     ).shuffle_data_sources(SCREAMING_SNAKE_CASE     )
            assert shuffled_it.n_shards == 3
            for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE     ):
                  lowerCAmelCase		,						lowerCAmelCase    		=			expected_row_ids_and_row_dicts[i]
                  assert row_id == expected_row_id
                  assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(20     ).repartition(4     )

      # Partitions 0 and 2
      lowerCAmelCase    		=			SparkExamplesIterable(SCREAMING_SNAKE_CASE     ).shard_data_sources(worker_id=0				,		num_workers=2     )
      assert shard_it_a.n_shards == 2
      lowerCAmelCase    		=			_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE				,		[0, 2]     )
      for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE     ):
            lowerCAmelCase		,						lowerCAmelCase    		=			expected_row_ids_and_row_dicts_a[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict

      # Partitions 1 and 3
      lowerCAmelCase    		=			SparkExamplesIterable(SCREAMING_SNAKE_CASE     ).shard_data_sources(worker_id=1				,		num_workers=2     )
      assert shard_it_a.n_shards == 2
      lowerCAmelCase    		=			_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE				,		[1, 3]     )
      for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE     ):
            lowerCAmelCase		,						lowerCAmelCase    		=			expected_row_ids_and_row_dicts_a[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			pyspark.sql.SparkSession.builder.master("""local[*]"""     ).appName("""pyspark"""     ).getOrCreate()
      lowerCAmelCase    		=			spark.range(1_00     ).repartition(1     )
      lowerCAmelCase    		=			Spark(SCREAMING_SNAKE_CASE     )
      # Choose a small max_shard_size for maximum partitioning.
      spark_builder._repartition_df_if_needed(max_shard_size=1     )
      # The new number of partitions should not be greater than the number of rows.
      assert spark_builder.df.rdd.getNumPartitions() == 1_00













"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =  {
    "uclanlp/visualbert-vqa": "https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json",
    "uclanlp/visualbert-vqa-pre": "https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json",
    "uclanlp/visualbert-vqa-coco-pre": (
        "https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json"
    ),
    "uclanlp/visualbert-vcr": "https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json",
    "uclanlp/visualbert-vcr-pre": "https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json",
    "uclanlp/visualbert-vcr-coco-pre": (
        "https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json"
    ),
    "uclanlp/visualbert-nlvr2": "https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json",
    "uclanlp/visualbert-nlvr2-pre": "https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json",
    "uclanlp/visualbert-nlvr2-coco-pre": (
        "https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json"
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}

class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'visual_bert'





      def __init__(			self ,		lowercase=30_522 ,		lowercase=768 ,		lowercase=512 ,		lowercase=12 ,		lowercase=12 ,		lowercase=3_072 ,		lowercase="gelu" ,		lowercase=0.1 ,		lowercase=0.1 ,		lowercase=512 ,		lowercase=2 ,		lowercase=0.02 ,		lowercase=1e-12 ,		lowercase=False ,		lowercase=True ,		lowercase=1 ,		lowercase=0 ,		lowercase=2 ,		**lowercase ,		)						->	int:
            super().__init__(pad_token_id=lowercase ,		bos_token_id=lowercase ,		eos_token_id=lowercase ,		**lowercase					)

            lowerCAmelCase    		=			vocab_size
            lowerCAmelCase    		=			max_position_embeddings
            lowerCAmelCase    		=			hidden_size
            lowerCAmelCase    		=			visual_embedding_dim
            lowerCAmelCase    		=			num_hidden_layers
            lowerCAmelCase    		=			num_attention_heads
            lowerCAmelCase    		=			intermediate_size
            lowerCAmelCase    		=			hidden_act
            lowerCAmelCase    		=			hidden_dropout_prob
            lowerCAmelCase    		=			attention_probs_dropout_prob
            lowerCAmelCase    		=			initializer_range
            lowerCAmelCase    		=			type_vocab_size
            lowerCAmelCase    		=			layer_norm_eps
            lowerCAmelCase    		=			bypass_transformer
            lowerCAmelCase    		=			special_visual_initialize






