


from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def 				UpperCAmelCase					(  UpperCAmelCase					,UpperCAmelCase   )->		Optional[int]:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							[]
					for part_id in partition_order:
										SCREAMING_SNAKE_CASE_						   =							df.where(f'''SPARK_PARTITION_ID() = {part_id}'''   ).collect()
										for row_idx, row in enumerate(__snake_case   ):
															expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict())   )
					return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		str:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(100   ).repartition(1   )
					SCREAMING_SNAKE_CASE_						   =							Spark(__snake_case   )
					# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
					# that each partition can hold 2 rows.
					spark_builder._repartition_df_if_needed(max_shard_size=16   )
					# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
					assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		Dict:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(10   ).repartition(2   )
					SCREAMING_SNAKE_CASE_						   =							[1, 0]
					SCREAMING_SNAKE_CASE_						   =							_generate_iterable_examples(__snake_case					,__snake_case   )  # Reverse the partitions.
					SCREAMING_SNAKE_CASE_						   =							_get_expected_row_ids_and_row_dicts_for_partition_order(__snake_case					,__snake_case   )

					for i, (row_id, row_dict) in enumerate(generate_fn()   ):
										SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							expected_row_ids_and_row_dicts[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		Optional[int]:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(10   ).repartition(1   )
					SCREAMING_SNAKE_CASE_						   =							SparkExamplesIterable(__snake_case   )
					assert it.n_shards == 1
					for i, (row_id, row_dict) in enumerate(__snake_case   ):
										assert row_id == f'''0_{i}'''
										assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		Optional[int]:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(30   ).repartition(3   )
					# Mock the generator so that shuffle reverses the partition indices.
					with patch('''numpy.random.Generator'''   ) as generator_mock:
										SCREAMING_SNAKE_CASE_						   =							lambda UpperCAmelCase   : x.reverse()
										SCREAMING_SNAKE_CASE_						   =							_get_expected_row_ids_and_row_dicts_for_partition_order(__snake_case					,[2, 1, 0]   )

										SCREAMING_SNAKE_CASE_						   =							SparkExamplesIterable(__snake_case   ).shuffle_data_sources(__snake_case   )
										assert shuffled_it.n_shards == 3
										for i, (row_id, row_dict) in enumerate(__snake_case   ):
															SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							expected_row_ids_and_row_dicts[i]
															assert row_id == expected_row_id
															assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		Optional[Any]:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(20   ).repartition(4   )

					# Partitions 0 and 2
					SCREAMING_SNAKE_CASE_						   =							SparkExamplesIterable(__snake_case   ).shard_data_sources(worker_id=0					,num_workers=2   )
					assert shard_it_a.n_shards == 2
					SCREAMING_SNAKE_CASE_						   =							_get_expected_row_ids_and_row_dicts_for_partition_order(__snake_case					,[0, 2]   )
					for i, (row_id, row_dict) in enumerate(__snake_case   ):
										SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							expected_row_ids_and_row_dicts_a[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict

					# Partitions 1 and 3
					SCREAMING_SNAKE_CASE_						   =							SparkExamplesIterable(__snake_case   ).shard_data_sources(worker_id=1					,num_workers=2   )
					assert shard_it_a.n_shards == 2
					SCREAMING_SNAKE_CASE_						   =							_get_expected_row_ids_and_row_dicts_for_partition_order(__snake_case					,[1, 3]   )
					for i, (row_id, row_dict) in enumerate(__snake_case   ):
										SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_						   =							expected_row_ids_and_row_dicts_a[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 				UpperCAmelCase					(  )->		Tuple:







					'''simple docstring'''



					SCREAMING_SNAKE_CASE_						   =							pyspark.sql.SparkSession.builder.master('''local[*]'''   ).appName('''pyspark'''   ).getOrCreate()
					SCREAMING_SNAKE_CASE_						   =							spark.range(100   ).repartition(1   )
					SCREAMING_SNAKE_CASE_						   =							Spark(__snake_case   )
					# Choose a small max_shard_size for maximum partitioning.
					spark_builder._repartition_df_if_needed(max_shard_size=1   )
					# The new number of partitions should not be greater than the number of rows.
					assert spark_builder.df.rdd.getNumPartitions() == 100



'''simple docstring'''






import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
   import torch







@is_pipeline_test
class lowercase_       (		unittest.TestCase			):







   """simple docstring"""

   __lowerCAmelCase         =   MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
   __lowerCAmelCase         =   TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
   def 	__UpperCAmelCase   (      self      :			Any, UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[int], UpperCamelCase__      :			Any     )  ->			List[str]:
        _A    		=  TextaTextGenerationPipeline(model=UpperCamelCase__, tokenizer=UpperCamelCase__     )
        return generator, ["Something to write", "Something else"]
   def 	__UpperCAmelCase   (      self      :			Any, UpperCamelCase__      :			Tuple, UpperCamelCase__      :			Optional[Any]     )  ->			List[str]:
        _A    		=  generator('Something there'     )
        self.assertEqual(UpperCamelCase__, [{'generated_text': ANY(UpperCamelCase__     )}]     )
        # These are encoder decoder, they don't just append to incoming string
        self.assertFalse(outputs[0]['generated_text'].startswith('Something there'     )     )

        _A    		=  generator(['This is great !', 'Something else'], num_return_sequences=2, do_sample=UpperCamelCase__     )
        self.assertEqual(
            UpperCamelCase__, [
                [{'generated_text': ANY(UpperCamelCase__     )}, {'generated_text': ANY(UpperCamelCase__     )}],
                [{'generated_text': ANY(UpperCamelCase__     )}, {'generated_text': ANY(UpperCamelCase__     )}],
            ], )

        _A    		=  generator(
            ['This is great !', 'Something else'], num_return_sequences=2, batch_size=2, do_sample=UpperCamelCase__     )
        self.assertEqual(
            UpperCamelCase__, [
                [{'generated_text': ANY(UpperCamelCase__     )}, {'generated_text': ANY(UpperCamelCase__     )}],
                [{'generated_text': ANY(UpperCamelCase__     )}, {'generated_text': ANY(UpperCamelCase__     )}],
            ], )

        with self.assertRaises(UpperCamelCase__     ):
             generator(4     )
   @require_torch
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			Optional[Any]:
        _A    		=  pipeline('text2text-generation', model='patrickvonplaten/t5-tiny-random', framework='pt'     )
        # do_sample=False necessary for reproducibility
        _A    		=  generator('Something there', do_sample=UpperCamelCase__     )
        self.assertEqual(UpperCamelCase__, [{'generated_text': ''}]     )

        _A    		=  3
        _A    		=  generator(
            'Something there', num_return_sequences=UpperCamelCase__, num_beams=UpperCamelCase__, )
        _A    		=  [
            {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide Beide'},
            {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide'},
            {'generated_text': ''},
        ]
        self.assertEqual(UpperCamelCase__, UpperCamelCase__     )

        _A    		=  generator('This is a test', do_sample=UpperCamelCase__, num_return_sequences=2, return_tensors=UpperCamelCase__     )
        self.assertEqual(
            UpperCamelCase__, [
                {'generated_token_ids': ANY(torch.Tensor     )},
                {'generated_token_ids': ANY(torch.Tensor     )},
            ], )
        _A    		=  generator.model.config.eos_token_id
        _A    		=  '<pad>'
        _A    		=  generator(
            ['This is a test', 'This is a second test'], do_sample=UpperCamelCase__, num_return_sequences=2, batch_size=2, return_tensors=UpperCamelCase__, )
        self.assertEqual(
            UpperCamelCase__, [
                [
                    {'generated_token_ids': ANY(torch.Tensor     )},
                    {'generated_token_ids': ANY(torch.Tensor     )},
                ],
                [
                    {'generated_token_ids': ANY(torch.Tensor     )},
                    {'generated_token_ids': ANY(torch.Tensor     )},
                ],
            ], )

   @require_tf
   def 	__UpperCAmelCase   (      self      :			Optional[Any]     )  ->			Optional[int]:
        _A    		=  pipeline('text2text-generation', model='patrickvonplaten/t5-tiny-random', framework='tf'     )
        # do_sample=False necessary for reproducibility
        _A    		=  generator('Something there', do_sample=UpperCamelCase__     )
        self.assertEqual(UpperCamelCase__, [{'generated_text': ''}]     )



