


'''simple docstring'''




from collections import UserDict
from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
       from PIL import Image

       from ..image_utils import load_image

if is_torch_available():
       from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING

if is_tf_available():
       from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
       from ..tf_utils import stable_softmax

_a   :   List[Any]												=   logging.get_logger(__name__)




@add_end_docstrings(__lowercase							)
class 			_lowercase (	__lowercase							):
   def __init__(						self       :      Any       ,   **SCREAMING_SNAKE_CASE_       :      Union[str, Any]      ) ->							List[Any]:
        super().__init__(**SCREAMING_SNAKE_CASE_      )

        requires_backends(self       ,   'vision'      )
        self.check_model_type(
            TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING
            if self.framework == 'tf'
            else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING      )
   def __call__(						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      Union[str, List[str], "Image", List["Image"]]       ,   **SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							int:
        return super().__call__(SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      List[Any]       ,   **SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							List[str]:
        __snake_case           =						{}
        if "candidate_labels" in kwargs:
             __snake_case           =						kwargs['candidate_labels']
        if "hypothesis_template" in kwargs:
             __snake_case           =						kwargs['hypothesis_template']

        return preprocess_params, {}, {}
   def a  (						self       :      str       ,   SCREAMING_SNAKE_CASE_       :      Tuple       ,   SCREAMING_SNAKE_CASE_       :      Tuple=None       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]="This is a photo of {}."      ) ->							Dict:
        __snake_case           =						load_image(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.image_processor(images=[image]       ,   return_tensors=self.framework      )
        __snake_case           =						candidate_labels
        __snake_case           =						[hypothesis_template.format(SCREAMING_SNAKE_CASE_      ) for x in candidate_labels]
        __snake_case           =						self.tokenizer(SCREAMING_SNAKE_CASE_       ,   return_tensors=self.framework       ,   padding=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						[text_inputs]
        return inputs
   def a  (						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      List[Any]      ) ->							Union[str, Any]:
        __snake_case           =						model_inputs.pop('candidate_labels'      )
        __snake_case           =						model_inputs.pop('text_inputs'      )
        if isinstance(text_inputs[0]       ,   SCREAMING_SNAKE_CASE_      ):
             __snake_case           =						text_inputs[0]
        else:
             # Batching case.
             __snake_case           =						text_inputs[0][0]

        __snake_case           =						self.model(**SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )

        __snake_case           =						{
            'candidate_labels': candidate_labels,
            'logits': outputs.logits_per_image,
        }
        return model_outputs





   def a  (						self       :      Tuple       ,   SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							List[str]:
        __snake_case           =						model_outputs.pop('candidate_labels'      )
        __snake_case           =						model_outputs['logits'][0]
        if self.framework == "pt":
             __snake_case           =						logits.softmax(dim=-1      ).squeeze(-1      )
             __snake_case           =						probs.tolist()
             if not isinstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      ):
                  __snake_case           =						[scores]
        elif self.framework == "tf":
             __snake_case           =						stable_softmax(SCREAMING_SNAKE_CASE_       ,   axis=-1      )
             __snake_case           =						probs.numpy().tolist()
        else:
             raise ValueError(f'Unsupported framework: {self.framework}'      )

        __snake_case           =						[
            {'score': score, 'label': candidate_label}
            for score, candidate_label in sorted(zip(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )       ,   key=lambda SCREAMING_SNAKE_CASE_      : -x[0]      )
        ]
        return result






'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def      UpperCamelCase_				(      A__  ,      A__						):
	a_      				=						[]
	for part_id in partition_order:
		a_      				=						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
		for row_idx, row in enumerate(A__						):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16						)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(2						)
	a_      				=						[1, 0]
	a_      				=						_generate_iterable_examples(A__  ,      A__						)  # Reverse the partitions.
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      A__						)

	for i, (row_id, row_dict) in enumerate(generate_fn()						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(1						)
	a_      				=						SparkExamplesIterable(A__						)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(A__						):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(30						).repartition(3						)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator"""						) as generator_mock:
		a_      				=						lambda A__						: x.reverse()
		a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [2, 1, 0]						)

		a_      				=						SparkExamplesIterable(A__						).shuffle_data_sources(A__						)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(A__						):
			a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(20						).repartition(4						)

	# Partitions 0 and 2
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=0  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [0, 2]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=1  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [1, 3]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1						)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00
