




"""simple docstring"""







import inspect
import unittest

from transformers import DecisionTransformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import DecisionTransformerModel
  from transformers.models.decision_transformer.modeling_decision_transformer import (
      DECISION_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
  )

class 				snake_case_			:







   """simple docstring"""




   def __init__(		self       ,					lowerCamelCase_       ,					lowerCamelCase_=1_3       ,					lowerCamelCase_=7       ,					lowerCamelCase_=6       ,					lowerCamelCase_=1_7       ,					lowerCamelCase_=2_3       ,					lowerCamelCase_=1_1       ,					lowerCamelCase_=True       ,					)				->   Dict:
       UpperCamelCase              =	parent
       UpperCamelCase              =	batch_size
       UpperCamelCase              =	seq_length
       UpperCamelCase              =	act_dim
       UpperCamelCase              =	state_dim
       UpperCamelCase              =	hidden_size
       UpperCamelCase              =	max_length
       UpperCamelCase              =	is_training







   def   UpperCAmelCase__			(		self)				->   str:
       UpperCamelCase              =	floats_tensor((self.batch_size, self.seq_length, self.state_dim))
       UpperCamelCase              =	floats_tensor((self.batch_size, self.seq_length, self.act_dim))
       UpperCamelCase              =	floats_tensor((self.batch_size, self.seq_length, 1))
       UpperCamelCase              =	floats_tensor((self.batch_size, self.seq_length, 1))
       UpperCamelCase              =	ids_tensor((self.batch_size, self.seq_length)       ,					vocab_size=1_0_0_0)
       UpperCamelCase              =	random_attention_mask((self.batch_size, self.seq_length))

       UpperCamelCase              =	self.get_config()

       return (
           config,
           states,
           actions,
           rewards,
           returns_to_go,
           timesteps,
           attention_mask,
       )







   def   UpperCAmelCase__			(		self)				->   str:
       return DecisionTransformerConfig(
           batch_size=self.batch_size       ,					seq_length=self.seq_length       ,					act_dim=self.act_dim       ,					state_dim=self.state_dim       ,					hidden_size=self.hidden_size       ,					max_length=self.max_length       ,					)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					)				->   Tuple:
       UpperCamelCase              =	DecisionTransformerModel(config=lowerCamelCase_)
       model.to(lowerCamelCase_)
       model.eval()
       UpperCamelCase              =	model(lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_)

       self.parent.assertEqual(result.state_preds.shape       ,					states.shape)
       self.parent.assertEqual(result.action_preds.shape       ,					actions.shape)
       self.parent.assertEqual(result.return_preds.shape       ,					returns_to_go.shape)
       self.parent.assertEqual(
           result.last_hidden_state.shape       ,					(self.batch_size, self.seq_length * 3, self.hidden_size))  # seq length *3 as there are 3 modelities: states, returns and actions







   def   UpperCAmelCase__			(		self)				->   int:
       UpperCamelCase              =	self.prepare_config_and_inputs()
       (
           (
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	(
           UpperCamelCase
       )       ,	
       )              =	config_and_inputs
       UpperCamelCase              =	{
           '''states''': states,
           '''actions''': actions,
           '''rewards''': rewards,
           '''returns_to_go''': returns_to_go,
           '''timesteps''': timesteps,
           '''attention_mask''': attention_mask,
       }
       return config, inputs_dict

@require_torch
class 				snake_case_			(					lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_	, unittest.TestCase			):







   """simple docstring"""




   A_				      =       (DecisionTransformerModel,) if is_torch_available() else ()
   A_				      =       ()
   A_				      =       {'''feature-extraction''': DecisionTransformerModel} if is_torch_available() else {}

   # Ignoring of a failing test from GenerationTesterMixin, as the model does not use inputs_ids
   A_				      =       False

   # Ignoring of a failing tests from ModelTesterMixin, as the model does not implement these features
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False
   A_				      =       False







   def   UpperCAmelCase__			(		self)				->   Any:
       UpperCamelCase              =	DecisionTransformerModelTester(self)
       UpperCamelCase              =	ConfigTester(self       ,					config_class=lowerCamelCase_       ,					hidden_size=3_7)







   def   UpperCAmelCase__			(		self)				->   str:
       self.config_tester.run_common_tests()







   def   UpperCAmelCase__			(		self)				->   Union[str, Any]:
       UpperCamelCase              =	self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCamelCase_)







   @slow
   def   UpperCAmelCase__			(		self)				->   Optional[int]:
       for model_name in DECISION_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           UpperCamelCase              =	DecisionTransformerModel.from_pretrained(lowerCamelCase_)
           self.assertIsNotNone(lowerCamelCase_)







   def   UpperCAmelCase__			(		self)				->   str:
       UpperCamelCase       ,	UpperCamelCase              =	self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
           UpperCamelCase              =	model_class(lowerCamelCase_)
           UpperCamelCase              =	inspect.signature(model.forward)
           # signature.parameters is an OrderedDict => so arg_names order is deterministic
           UpperCamelCase              =	[*signature.parameters.keys()]

           UpperCamelCase              =	[
               '''states''',
               '''actions''',
               '''rewards''',
               '''returns_to_go''',
               '''timesteps''',
               '''attention_mask''',
           ]

           self.assertListEqual(arg_names[: len(lowerCamelCase_)]       ,					lowerCamelCase_)

@require_torch
class 				snake_case_			(					unittest.TestCase			):







   """simple docstring"""




   @slow
   def   UpperCAmelCase__			(		self)				->   int:
       UpperCamelCase              =	2  # number of steps of autoregressive prediction we will perform
       UpperCamelCase              =	1_0  # defined by the RL environment, may be normalized
       UpperCamelCase              =	DecisionTransformerModel.from_pretrained('''edbeeching/decision-transformer-gym-hopper-expert''')
       UpperCamelCase              =	model.to(lowerCamelCase_)
       UpperCamelCase              =	model.config
       torch.manual_seed(0)
       UpperCamelCase              =	torch.randn(1       ,					1       ,					config.state_dim).to(device=lowerCamelCase_       ,					dtype=torch.floataa)  # env.reset()

       UpperCamelCase              =	torch.tensor(
           [[0.24_2793, -0.2869_3074, 0.874_2613], [0.6781_5274, -0.0810_1085, -0.1295_2147]]       ,					device=lowerCamelCase_)

       UpperCamelCase              =	torch.tensor(lowerCamelCase_       ,					device=lowerCamelCase_       ,					dtype=torch.floataa).reshape(1       ,					1       ,					1)
       UpperCamelCase              =	state
       UpperCamelCase              =	torch.zeros(1       ,					0       ,					config.act_dim       ,					device=lowerCamelCase_       ,					dtype=torch.floataa)
       UpperCamelCase              =	torch.zeros(1       ,					0       ,					device=lowerCamelCase_       ,					dtype=torch.floataa)
       UpperCamelCase              =	torch.tensor(0       ,					device=lowerCamelCase_       ,					dtype=torch.long).reshape(1       ,					1)

       for step in range(lowerCamelCase_):
           UpperCamelCase              =	torch.cat([actions, torch.zeros(1       ,					1       ,					config.act_dim       ,					device=lowerCamelCase_)]       ,					dim=1)
           UpperCamelCase              =	torch.cat([rewards, torch.zeros(1       ,					1       ,					device=lowerCamelCase_)]       ,					dim=1)

           UpperCamelCase              =	torch.ones(1       ,					states.shape[1]).to(dtype=torch.long       ,					device=states.device)

           with torch.no_grad():
               UpperCamelCase       ,	UpperCamelCase       ,	UpperCamelCase              =	model(
                   states=lowerCamelCase_       ,					actions=lowerCamelCase_       ,					rewards=lowerCamelCase_       ,					returns_to_go=lowerCamelCase_       ,					timesteps=lowerCamelCase_       ,					attention_mask=lowerCamelCase_       ,					return_dict=lowerCamelCase_       ,					)

           self.assertEqual(action_pred.shape       ,					actions.shape)
           self.assertTrue(torch.allclose(action_pred[0, -1]       ,					expected_outputs[step]       ,					atol=1e-4))
           UpperCamelCase       ,	UpperCamelCase       ,	UpperCamelCase       ,	UpperCamelCase              =	(  # env.step(action)
               torch.randn(1       ,					1       ,					config.state_dim).to(device=lowerCamelCase_       ,					dtype=torch.floataa),
               1.0,
               False,
               {},
           )

           UpperCamelCase              =	action_pred[0, -1]
           UpperCamelCase              =	torch.cat([states, state]       ,					dim=1)
           UpperCamelCase              =	returns_to_go[0, -1] - reward
           UpperCamelCase              =	torch.cat([returns_to_go, pred_return.reshape(1       ,					1       ,					1)]       ,					dim=1)
           UpperCamelCase              =	torch.cat(
               [timesteps, torch.ones((1, 1)       ,					device=lowerCamelCase_       ,					dtype=torch.long) * (step + 1)]       ,					dim=1)

'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def      UpperCamelCase_				(      A__  ,      A__						):
	a_      				=						[]
	for part_id in partition_order:
		a_      				=						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
		for row_idx, row in enumerate(A__						):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16						)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(2						)
	a_      				=						[1, 0]
	a_      				=						_generate_iterable_examples(A__  ,      A__						)  # Reverse the partitions.
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      A__						)

	for i, (row_id, row_dict) in enumerate(generate_fn()						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(1						)
	a_      				=						SparkExamplesIterable(A__						)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(A__						):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(30						).repartition(3						)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator"""						) as generator_mock:
		a_      				=						lambda A__						: x.reverse()
		a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [2, 1, 0]						)

		a_      				=						SparkExamplesIterable(A__						).shuffle_data_sources(A__						)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(A__						):
			a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(20						).repartition(4						)

	# Partitions 0 and 2
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=0  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [0, 2]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=1  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [1, 3]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1						)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00
