





'''simple docstring'''

from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def    a__							(							lowercase   :	Optional[Any],   lowercase   :	Dict )						->						List[str]:





     """simple docstring"""

     _UpperCamelCase							       =			[]
     for part_id in partition_order:
          _UpperCamelCase							       =			df.where(F"""SPARK_PARTITION_ID() = {part_id}""" ).collect()
          for row_idx, row in enumerate(lowercase ):
               expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict()) )
     return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						Any:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(100 ).repartition(1 )
     _UpperCamelCase							       =			Spark(lowercase )
     # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
     # that each partition can hold 2 rows.
     spark_builder._repartition_df_if_needed(max_shard_size=16 )
     # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
     assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						Optional[int]:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(10 ).repartition(2 )
     _UpperCamelCase							       =			[1, 0]
     _UpperCamelCase							       =			_generate_iterable_examples(lowercase,   lowercase )  # Reverse the partitions.
     _UpperCamelCase							       =			_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase,   lowercase )

     for i, (row_id, row_dict) in enumerate(generate_fn() ):
          _UpperCamelCase       ,							_UpperCamelCase							       =			expected_row_ids_and_row_dicts[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						Dict:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(10 ).repartition(1 )
     _UpperCamelCase							       =			SparkExamplesIterable(lowercase )
     assert it.n_shards == 1
     for i, (row_id, row_dict) in enumerate(lowercase ):
          assert row_id == F"""0_{i}"""
          assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						Tuple:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(30 ).repartition(3 )
     # Mock the generator so that shuffle reverses the partition indices.
     with patch('''numpy.random.Generator''' ) as generator_mock:
          _UpperCamelCase							       =			lambda lowercase : x.reverse()
          _UpperCamelCase							       =			_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase,   [2, 1, 0] )

          _UpperCamelCase							       =			SparkExamplesIterable(lowercase ).shuffle_data_sources(lowercase )
          assert shuffled_it.n_shards == 3
          for i, (row_id, row_dict) in enumerate(lowercase ):
               _UpperCamelCase       ,							_UpperCamelCase							       =			expected_row_ids_and_row_dicts[i]
               assert row_id == expected_row_id
               assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						int:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(20 ).repartition(4 )

     # Partitions 0 and 2
     _UpperCamelCase							       =			SparkExamplesIterable(lowercase ).shard_data_sources(worker_id=0,   num_workers=2 )
     assert shard_it_a.n_shards == 2
     _UpperCamelCase							       =			_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase,   [0, 2] )
     for i, (row_id, row_dict) in enumerate(lowercase ):
          _UpperCamelCase       ,							_UpperCamelCase							       =			expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

     # Partitions 1 and 3
     _UpperCamelCase							       =			SparkExamplesIterable(lowercase ).shard_data_sources(worker_id=1,   num_workers=2 )
     assert shard_it_a.n_shards == 2
     _UpperCamelCase							       =			_get_expected_row_ids_and_row_dicts_for_partition_order(lowercase,   [1, 3] )
     for i, (row_id, row_dict) in enumerate(lowercase ):
          _UpperCamelCase       ,							_UpperCamelCase							       =			expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def    a__							(							)						->						int:





     """simple docstring"""

     _UpperCamelCase							       =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
     _UpperCamelCase							       =			spark.range(100 ).repartition(1 )
     _UpperCamelCase							       =			Spark(lowercase )
     # Choose a small max_shard_size for maximum partitioning.
     spark_builder._repartition_df_if_needed(max_shard_size=1 )
     # The new number of partitions should not be greater than the number of rows.
     assert spark_builder.df.rdd.getNumPartitions() == 100










'''simple docstring'''

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..models.auto import AutoModelForVisionaSeq
from ..utils import requires_backends
from .base import PipelineTool


if TYPE_CHECKING:
   from PIL import Image
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	str									=       'Salesforce/blip-image-captioning-base'
       _snake_case      :	Union[str, Any]									=       (
           'This is a tool that generates a description of an image. It takes an input named `image` which should be the '
           'image to caption, and returns a text that contains the description in English.'
       )
       _snake_case      :	List[Any]									=       'image_captioner'
       _snake_case      :	Union[str, Any]									=       AutoModelForVisionaSeq

       _snake_case      :	Dict									=       ['image']
       _snake_case      :	Optional[int]									=       ['text']





       def __init__(				self  :						Union[str, Any]			,					*lowerCAmelCase__  :						Optional[Any]			,					**lowerCAmelCase__  :						Optional[int]	)       ->	Tuple:

            '''simple docstring'''




            requires_backends(self			,					['''vision''']	)
            super().__init__(*lowerCAmelCase__			,					**lowerCAmelCase__	)





       def snake_case__		(				self  :						Tuple			,					lowerCAmelCase__  :						"Image"	)       ->	Optional[int]:

            '''simple docstring'''




            return self.pre_processor(images=lowerCAmelCase__			,					return_tensors='''pt'''	)





       def snake_case__		(				self  :						Any			,					lowerCAmelCase__  :						Union[str, Any]	)       ->	Union[str, Any]:

            '''simple docstring'''




            return self.model.generate(**lowerCAmelCase__	)







       def snake_case__		(				self  :						int			,					lowerCAmelCase__  :						List[str]	)       ->	List[str]:

            '''simple docstring'''




            return self.pre_processor.batch_decode(lowerCAmelCase__			,					skip_special_tokens=lowerCAmelCase__	)[0].strip()



