


'''simple docstring'''



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
	    OpenAIGPTConfig,
	    OpenAIGPTDoubleHeadsModel,
	    OpenAIGPTForSequenceClassification,
	    OpenAIGPTLMHeadModel,
	    OpenAIGPTModel,
	)





class 				__UpperCamelCase							:




	def __init__(  self		,							__a		,							__a=13		,							__a=7		,							__a=True		,							__a=True		,							__a=True		,							__a=99		,							__a=32		,							__a=5		,							__a=4		,							__a=37		,							__a="gelu"		,							__a=0.1		,							__a=0.1		,							__a=512		,							__a=16		,							__a=2		,							__a=0.02		,							__a=3		,							__a=4		,							__a=None		,							):

						'''simple docstring'''





						__a     :			str          =    parent
						__a     :			Optional[Any]          =    batch_size
						__a     :			Dict          =    seq_length
						__a     :			Union[str, Any]          =    is_training
						__a     :			Any          =    use_token_type_ids
						__a     :			Optional[Any]          =    use_labels
						__a     :			int          =    vocab_size
						__a     :			Optional[Any]          =    hidden_size
						__a     :			Optional[Any]          =    num_hidden_layers
						__a     :			str          =    num_attention_heads
						__a     :			int          =    intermediate_size
						__a     :			Any          =    hidden_act
						__a     :			Tuple          =    hidden_dropout_prob
						__a     :			List[Any]          =    attention_probs_dropout_prob
						__a     :			int          =    max_position_embeddings
						__a     :			Tuple          =    type_vocab_size
						__a     :			Optional[int]          =    type_sequence_label_size
						__a     :			str          =    initializer_range
						__a     :			List[str]          =    num_labels
						__a     :			Tuple          =    num_choices
						__a     :			Dict          =    scope
						__a     :			Optional[int]          =    self.vocab_size - 1




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			Optional[Any]          =    ids_tensor([self.batch_size, self.seq_length]		,							self.vocab_size				)

						__a     :			Union[str, Any]          =    None
						if self.use_token_type_ids:
											__a     :			List[Any]          =    ids_tensor([self.batch_size, self.seq_length]		,							self.type_vocab_size				)

						__a     :			Dict          =    None
						__a     :			Optional[int]          =    None
						__a     :			Dict          =    None
						if self.use_labels:
											__a     :			str          =    ids_tensor([self.batch_size]		,							self.type_sequence_label_size				)
											__a     :			List[Any]          =    ids_tensor([self.batch_size, self.seq_length]		,							self.num_labels				)
											__a     :			List[str]          =    ids_tensor([self.batch_size]		,							self.num_choices				)

						__a     :			Dict          =    OpenAIGPTConfig(
						    vocab_size=self.vocab_size		,							n_embd=self.hidden_size		,							n_layer=self.num_hidden_layers		,							n_head=self.num_attention_heads		,							n_positions=self.max_position_embeddings		,							pad_token_id=self.pad_token_id		,							)

						__a     :			str          =    ids_tensor([self.num_hidden_layers, self.num_attention_heads]		,							2				)

						return (
						    config,
						    input_ids,
						    head_mask,
						    token_type_ids,
						    sequence_labels,
						    token_labels,
						    choice_labels,
						)




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a		,							__a		,							*__a				):

						'''simple docstring'''





						__a     :			List[str]          =    OpenAIGPTModel(config=__a				)
						model.to(__a				)
						model.eval()

						__a     :			str          =    model(__a		,							token_type_ids=__a		,							head_mask=__a				)
						__a     :			List[Any]          =    model(__a		,							token_type_ids=__a				)
						__a     :			Dict          =    model(__a				)

						self.parent.assertEqual(result.last_hidden_state.shape		,							(self.batch_size, self.seq_length, self.hidden_size)				)




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a		,							__a		,							*__a				):

						'''simple docstring'''





						__a     :			Optional[int]          =    OpenAIGPTLMHeadModel(__a				)
						model.to(__a				)
						model.eval()

						__a     :			List[Any]          =    model(__a		,							token_type_ids=__a		,							labels=__a				)
						self.parent.assertEqual(result.loss.shape		,							()				)
						self.parent.assertEqual(result.logits.shape		,							(self.batch_size, self.seq_length, self.vocab_size)				)




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a		,							__a		,							*__a				):

						'''simple docstring'''





						__a     :			List[str]          =    OpenAIGPTDoubleHeadsModel(__a				)
						model.to(__a				)
						model.eval()

						__a     :			Optional[int]          =    model(__a		,							token_type_ids=__a		,							labels=__a				)
						self.parent.assertEqual(result.loss.shape		,							()				)
						self.parent.assertEqual(result.logits.shape		,							(self.batch_size, self.seq_length, self.vocab_size)				)




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a		,							__a		,							*__a				):

						'''simple docstring'''





						__a     :			int          =    self.num_labels
						__a     :			Dict          =    OpenAIGPTForSequenceClassification(__a				)
						model.to(__a				)
						model.eval()

						__a     :			Optional[int]          =    ids_tensor([self.batch_size]		,							self.type_sequence_label_size				)
						__a     :			Optional[int]          =    model(__a		,							token_type_ids=__a		,							labels=__a				)
						self.parent.assertEqual(result.logits.shape		,							(self.batch_size, self.num_labels)				)





	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			Optional[int]          =    self.prepare_config_and_inputs()
						(
						    (
						    __a  
						)       ,							(
						    __a  
						)       ,							(
						    __a  
						)       ,							(
						    __a  
						)       ,							(
						    __a  
						)       ,							(
						    __a  
						)       ,							(
						    __a  
						)       ,							  
						)     :			List[str]          =    config_and_inputs
						__a     :			Optional[int]          =    {
						    'input_ids': input_ids,
						    'token_type_ids': token_type_ids,
						    'head_mask': head_mask,
						}

						return config, inputs_dict





@require_torch
class 				__UpperCamelCase							(						UpperCamelCase__						, UpperCamelCase__						, UpperCamelCase__						, unittest.TestCase   ):
	A_           =       (
	    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
	    if is_torch_available()
	    else ()
	)
	A_           =       (
	    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
	)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
	A_           =       (
	    {
	        'feature-extraction': OpenAIGPTModel,
	        'text-classification': OpenAIGPTForSequenceClassification,
	        'text-generation': OpenAIGPTLMHeadModel,
	        'zero-shot': OpenAIGPTForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a		,							__a		,							__a				):

						'''simple docstring'''





						if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
											# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
											# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
											# tiny config could not be created.
											return True

						return False




	def  __UpperCAmelCase				(  self		,							__a		,							__a		,							__a=False				):

						'''simple docstring'''





						__a     :			int          =    super()._prepare_for_class(__a		,							__a		,							return_labels=__a				)

						if return_labels:
											if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
																__a     :			Dict          =    torch.zeros(
																    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)		,							dtype=torch.long		,							device=__a		,							)
																__a     :			Optional[int]          =    inputs_dict['labels']
																__a     :			Union[str, Any]          =    inputs_dict['labels']
																__a     :			int          =    torch.zeros(
																    (self.model_tester.batch_size, self.model_tester.num_choices)		,							dtype=torch.long		,							device=__a		,							)
																__a     :			Dict          =    torch.zeros(
																    self.model_tester.batch_size		,							dtype=torch.long		,							device=__a				)
						return inputs_dict




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			Tuple          =    OpenAIGPTModelTester(self				)
						__a     :			List[Any]          =    ConfigTester(self		,							config_class=__a		,							n_embd=37				)




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						self.config_tester.run_common_tests()




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			Tuple          =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_openai_gpt_model(*__a				)




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			List[str]          =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_lm_head_model(*__a				)




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			List[Any]          =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_double_lm_head_model(*__a				)




	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			int          =    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__a				)





	@slow
	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__a     :			List[str]          =    OpenAIGPTModel.from_pretrained(__a				)
											self.assertIsNotNone(__a				)





@require_torch
class 				__UpperCamelCase							(						unittest.TestCase   ):




	@slow
	def  __UpperCAmelCase				(  self				):

						'''simple docstring'''





						__a     :			Dict          =    OpenAIGPTLMHeadModel.from_pretrained('openai-gpt'				)
						model.to(__a				)
						__a     :			int          =    torch.tensor([[481, 4735, 544]]		,							dtype=torch.long		,							device=__a				)  # the president is
						__a     :			Any          =    [
						    481,
						    4735,
						    544,
						    246,
						    963,
						    870,
						    762,
						    239,
						    244,
						    4_0477,
						    244,
						    249,
						    719,
						    881,
						    487,
						    544,
						    240,
						    244,
						    603,
						    481,
						]  # the president is a very good man. " \n " i\'m sure he is, " said the

						__a     :			List[str]          =    model.generate(__a		,							do_sample=__a				)
						self.assertListEqual(output_ids[0].tolist()		,							__a				)





'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def      UpperCamelCase_				(      A__  ,      A__						):
	a_      				=						[]
	for part_id in partition_order:
		a_      				=						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
		for row_idx, row in enumerate(A__						):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16						)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(2						)
	a_      				=						[1, 0]
	a_      				=						_generate_iterable_examples(A__  ,      A__						)  # Reverse the partitions.
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      A__						)

	for i, (row_id, row_dict) in enumerate(generate_fn()						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(1						)
	a_      				=						SparkExamplesIterable(A__						)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(A__						):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(30						).repartition(3						)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator"""						) as generator_mock:
		a_      				=						lambda A__						: x.reverse()
		a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [2, 1, 0]						)

		a_      				=						SparkExamplesIterable(A__						).shuffle_data_sources(A__						)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(A__						):
			a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(20						).repartition(4						)

	# Partitions 0 and 2
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=0  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [0, 2]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=1  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [1, 3]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1						)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00
