prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  [rllib] _get_torch_exploration_action doesn't support tuple action dist
  <!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->
  ### System information
  
  *  **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Mac OS 10.15.4
  *  **Ray installed from (source or binary)**: binary (via pip)
  *  **Ray version**: 0.8.6., but nothing seems to have changed on master
  *  **Python version**: 3.7
  
  ### What is the problem?
  
  When using tuple action distributions (as advised in #6372) and exploration is disabled, the line:
  
  https://github.com/ray-project/ray/blob/a462ae2747afbeb9047e443cd51e67e3fe0b49e6/rllib/utils/exploration/stochastic_sampling.py#L75
  
  from `_get_torch_exploration_action` raises the following exception:
  
  ```
  AttributeError: 'tuple' object has no attribute 'size'
  ```
  
  A simple fix that supports any type of distribution would be:
  ```python
  logp = torch.zeros_like(action_dist.sampled_action_logp())
  ```
  
  I can submit a PR if it helps. 
  
  ### Reproduction (REQUIRED)
  
  Exact command to reproduce: python `rllib_cartpole.py` for the following file
  
  ```python
  import gym.envs.classic_control
  from gym.spaces import Tuple, Discrete
  
  import ray
  from ray import tune
  
  
  class CustomCartpole(gym.envs.classic_control.CartPoleEnv):
      """Add a dimension to the cartpole action space that is ignored."""
  
      def __init__(self, env_config):
          super().__init__()
          # if override_actions is false this is just the Cartpole environment
          self.override_actions = env_config['override_actions']
          if self.override_actions:
              # 2 is the environment's normal action space
              # 4 is just a dummy number to give it an extra dimension
              self.original_action_space = self.action_space
              self.action_space = Tuple([Discrete(2), Discrete(4)])
              self.tuple_action_space = self.action_space
  
      def step(self, action):
          # call the cartpole environment with the original action
          if self.override_actions:
              self.action_space = self.original_action_space
              return super().step(action[0])
          else:
              return super().step(action)
  
  
  def main():
      ray.init()
      tune.run(
          "PPO",
          stop={"episode_reward_mean": 50},
          config={
              "env": CustomCartpole,
              "env_config": {'override_actions': True},
              "num_gpus": 0,
              "num_workers": 1,
              "eager": False,
              "evaluation_interval": 1,
              "evaluation_config": {
                  "explore": False,
              },
              "framework": "torch",
          },
      )
  
  
  if __name__ == '__main__':
      main()
  ```
  
  
  - [x] I have verified my script runs in a clean environment and reproduces the issue.
  - [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `rllib/utils/exploration/stochastic_sampling.py`
  Content:
  ```
  1 import tree
  2 from typing import Union
  3 
  4 from ray.rllib.models.action_dist import ActionDistribution
  5 from ray.rllib.models.modelv2 import ModelV2
  6 from ray.rllib.utils.annotations import override
  7 from ray.rllib.utils.exploration.exploration import Exploration
  8 from ray.rllib.utils.framework import try_import_tf, try_import_torch, \
  9     TensorType
  10 
  11 tf1, tf, tfv = try_import_tf()
  12 torch, _ = try_import_torch()
  13 
  14 
  15 class StochasticSampling(Exploration):
  16     """An exploration that simply samples from a distribution.
  17 
  18     The sampling can be made deterministic by passing explore=False into
  19     the call to `get_exploration_action`.
  20     Also allows for scheduled parameters for the distributions, such as
  21     lowering stddev, temperature, etc.. over time.
  22     """
  23 
  24     def __init__(self, action_space, *, framework: str, model: ModelV2,
  25                  **kwargs):
  26         """Initializes a StochasticSampling Exploration object.
  27 
  28         Args:
  29             action_space (Space): The gym action space used by the environment.
  30             framework (str): One of None, "tf", "torch".
  31         """
  32         assert framework is not None
  33         super().__init__(
  34             action_space, model=model, framework=framework, **kwargs)
  35 
  36     @override(Exploration)
  37     def get_exploration_action(self,
  38                                *,
  39                                action_distribution: ActionDistribution,
  40                                timestep: Union[int, TensorType],
  41                                explore: bool = True):
  42         if self.framework == "torch":
  43             return self._get_torch_exploration_action(action_distribution,
  44                                                       explore)
  45         else:
  46             return self._get_tf_exploration_action_op(action_distribution,
  47                                                       explore)
  48 
  49     def _get_tf_exploration_action_op(self, action_dist, explore):
  50         sample = action_dist.sample()
  51         deterministic_sample = action_dist.deterministic_sample()
  52         action = tf.cond(
  53             tf.constant(explore) if isinstance(explore, bool) else explore,
  54             true_fn=lambda: sample,
  55             false_fn=lambda: deterministic_sample)
  56 
  57         def logp_false_fn():
  58             batch_size = tf.shape(tree.flatten(action)[0])[0]
  59             return tf.zeros(shape=(batch_size, ), dtype=tf.float32)
  60 
  61         logp = tf.cond(
  62             tf.constant(explore) if isinstance(explore, bool) else explore,
  63             true_fn=lambda: action_dist.sampled_action_logp(),
  64             false_fn=logp_false_fn)
  65 
  66         return action, logp
  67 
  68     @staticmethod
  69     def _get_torch_exploration_action(action_dist, explore):
  70         if explore:
  71             action = action_dist.sample()
  72             logp = action_dist.sampled_action_logp()
  73         else:
  74             action = action_dist.deterministic_sample()
  75             logp = torch.zeros((action.size()[0], ), dtype=torch.float32)
  76         return action, logp
  77
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/rllib/utils/exploration/stochastic_sampling.py b/rllib/utils/exploration/stochastic_sampling.py
  --- a/rllib/utils/exploration/stochastic_sampling.py
  +++ b/rllib/utils/exploration/stochastic_sampling.py
  @@ -72,5 +72,5 @@
               logp = action_dist.sampled_action_logp()
           else:
               action = action_dist.deterministic_sample()
  -            logp = torch.zeros((action.size()[0], ), dtype=torch.float32)
  +            logp = torch.zeros_like(action_dist.sampled_action_logp())
           return action, logp
  
verification_info: {"golden_diff": "diff --git a/rllib/utils/exploration/stochastic_sampling.py b/rllib/utils/exploration/stochastic_sampling.py\n--- a/rllib/utils/exploration/stochastic_sampling.py\n+++ b/rllib/utils/exploration/stochastic_sampling.py\n@@ -72,5 +72,5 @@\n             logp = action_dist.sampled_action_logp()\n         else:\n             action = action_dist.deterministic_sample()\n-            logp = torch.zeros((action.size()[0], ), dtype=torch.float32)\n+            logp = torch.zeros_like(action_dist.sampled_action_logp())\n         return action, logp\n", "issue": "[rllib] _get_torch_exploration_action doesn't support tuple action dist\n<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->\r\n### System information\r\n\r\n*  **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Mac OS 10.15.4\r\n*  **Ray installed from (source or binary)**: binary (via pip)\r\n*  **Ray version**: 0.8.6., but nothing seems to have changed on master\r\n*  **Python version**: 3.7\r\n\r\n### What is the problem?\r\n\r\nWhen using tuple action distributions (as advised in #6372) and exploration is disabled, the line:\r\n\r\nhttps://github.com/ray-project/ray/blob/a462ae2747afbeb9047e443cd51e67e3fe0b49e6/rllib/utils/exploration/stochastic_sampling.py#L75\r\n\r\nfrom `_get_torch_exploration_action` raises the following exception:\r\n\r\n```\r\nAttributeError: 'tuple' object has no attribute 'size'\r\n```\r\n\r\nA simple fix that supports any type of distribution would be:\r\n```python\r\nlogp = torch.zeros_like(action_dist.sampled_action_logp())\r\n```\r\n\r\nI can submit a PR if it helps. \r\n\r\n### Reproduction (REQUIRED)\r\n\r\nExact command to reproduce: python `rllib_cartpole.py` for the following file\r\n\r\n```python\r\nimport gym.envs.classic_control\r\nfrom gym.spaces import Tuple, Discrete\r\n\r\nimport ray\r\nfrom ray import tune\r\n\r\n\r\nclass CustomCartpole(gym.envs.classic_control.CartPoleEnv):\r\n    \"\"\"Add a dimension to the cartpole action space that is ignored.\"\"\"\r\n\r\n    def __init__(self, env_config):\r\n        super().__init__()\r\n        # if override_actions is false this is just the Cartpole environment\r\n        self.override_actions = env_config['override_actions']\r\n        if self.override_actions:\r\n            # 2 is the environment's normal action space\r\n            # 4 is just a dummy number to give it an extra dimension\r\n            self.original_action_space = self.action_space\r\n            self.action_space = Tuple([Discrete(2), Discrete(4)])\r\n            self.tuple_action_space = self.action_space\r\n\r\n    def step(self, action):\r\n        # call the cartpole environment with the original action\r\n        if self.override_actions:\r\n            self.action_space = self.original_action_space\r\n            return super().step(action[0])\r\n        else:\r\n            return super().step(action)\r\n\r\n\r\ndef main():\r\n    ray.init()\r\n    tune.run(\r\n        \"PPO\",\r\n        stop={\"episode_reward_mean\": 50},\r\n        config={\r\n            \"env\": CustomCartpole,\r\n            \"env_config\": {'override_actions': True},\r\n            \"num_gpus\": 0,\r\n            \"num_workers\": 1,\r\n            \"eager\": False,\r\n            \"evaluation_interval\": 1,\r\n            \"evaluation_config\": {\r\n                \"explore\": False,\r\n            },\r\n            \"framework\": \"torch\",\r\n        },\r\n    )\r\n\r\n\r\nif __name__ == '__main__':\r\n    main()\r\n```\r\n\r\n\r\n- [x] I have verified my script runs in a clean environment and reproduces the issue.\r\n- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).\r\n\n", "before_files": [{"content": "import tree\nfrom typing import Union\n\nfrom ray.rllib.models.action_dist import ActionDistribution\nfrom ray.rllib.models.modelv2 import ModelV2\nfrom ray.rllib.utils.annotations import override\nfrom ray.rllib.utils.exploration.exploration import Exploration\nfrom ray.rllib.utils.framework import try_import_tf, try_import_torch, \\\n    TensorType\n\ntf1, tf, tfv = try_import_tf()\ntorch, _ = try_import_torch()\n\n\nclass StochasticSampling(Exploration):\n    \"\"\"An exploration that simply samples from a distribution.\n\n    The sampling can be made deterministic by passing explore=False into\n    the call to `get_exploration_action`.\n    Also allows for scheduled parameters for the distributions, such as\n    lowering stddev, temperature, etc.. over time.\n    \"\"\"\n\n    def __init__(self, action_space, *, framework: str, model: ModelV2,\n                 **kwargs):\n        \"\"\"Initializes a StochasticSampling Exploration object.\n\n        Args:\n            action_space (Space): The gym action space used by the environment.\n            framework (str): One of None, \"tf\", \"torch\".\n        \"\"\"\n        assert framework is not None\n        super().__init__(\n            action_space, model=model, framework=framework, **kwargs)\n\n    @override(Exploration)\n    def get_exploration_action(self,\n                               *,\n                               action_distribution: ActionDistribution,\n                               timestep: Union[int, TensorType],\n                               explore: bool = True):\n        if self.framework == \"torch\":\n            return self._get_torch_exploration_action(action_distribution,\n                                                      explore)\n        else:\n            return self._get_tf_exploration_action_op(action_distribution,\n                                                      explore)\n\n    def _get_tf_exploration_action_op(self, action_dist, explore):\n        sample = action_dist.sample()\n        deterministic_sample = action_dist.deterministic_sample()\n        action = tf.cond(\n            tf.constant(explore) if isinstance(explore, bool) else explore,\n            true_fn=lambda: sample,\n            false_fn=lambda: deterministic_sample)\n\n        def logp_false_fn():\n            batch_size = tf.shape(tree.flatten(action)[0])[0]\n            return tf.zeros(shape=(batch_size, ), dtype=tf.float32)\n\n        logp = tf.cond(\n            tf.constant(explore) if isinstance(explore, bool) else explore,\n            true_fn=lambda: action_dist.sampled_action_logp(),\n            false_fn=logp_false_fn)\n\n        return action, logp\n\n    @staticmethod\n    def _get_torch_exploration_action(action_dist, explore):\n        if explore:\n            action = action_dist.sample()\n            logp = action_dist.sampled_action_logp()\n        else:\n            action = action_dist.deterministic_sample()\n            logp = torch.zeros((action.size()[0], ), dtype=torch.float32)\n        return action, logp\n", "path": "rllib/utils/exploration/stochastic_sampling.py"}], "after_files": [{"content": "import tree\nfrom typing import Union\n\nfrom ray.rllib.models.action_dist import ActionDistribution\nfrom ray.rllib.models.modelv2 import ModelV2\nfrom ray.rllib.utils.annotations import override\nfrom ray.rllib.utils.exploration.exploration import Exploration\nfrom ray.rllib.utils.framework import try_import_tf, try_import_torch, \\\n    TensorType\n\ntf1, tf, tfv = try_import_tf()\ntorch, _ = try_import_torch()\n\n\nclass StochasticSampling(Exploration):\n    \"\"\"An exploration that simply samples from a distribution.\n\n    The sampling can be made deterministic by passing explore=False into\n    the call to `get_exploration_action`.\n    Also allows for scheduled parameters for the distributions, such as\n    lowering stddev, temperature, etc.. over time.\n    \"\"\"\n\n    def __init__(self, action_space, *, framework: str, model: ModelV2,\n                 **kwargs):\n        \"\"\"Initializes a StochasticSampling Exploration object.\n\n        Args:\n            action_space (Space): The gym action space used by the environment.\n            framework (str): One of None, \"tf\", \"torch\".\n        \"\"\"\n        assert framework is not None\n        super().__init__(\n            action_space, model=model, framework=framework, **kwargs)\n\n    @override(Exploration)\n    def get_exploration_action(self,\n                               *,\n                               action_distribution: ActionDistribution,\n                               timestep: Union[int, TensorType],\n                               explore: bool = True):\n        if self.framework == \"torch\":\n            return self._get_torch_exploration_action(action_distribution,\n                                                      explore)\n        else:\n            return self._get_tf_exploration_action_op(action_distribution,\n                                                      explore)\n\n    def _get_tf_exploration_action_op(self, action_dist, explore):\n        sample = action_dist.sample()\n        deterministic_sample = action_dist.deterministic_sample()\n        action = tf.cond(\n            tf.constant(explore) if isinstance(explore, bool) else explore,\n            true_fn=lambda: sample,\n            false_fn=lambda: deterministic_sample)\n\n        def logp_false_fn():\n            batch_size = tf.shape(tree.flatten(action)[0])[0]\n            return tf.zeros(shape=(batch_size, ), dtype=tf.float32)\n\n        logp = tf.cond(\n            tf.constant(explore) if isinstance(explore, bool) else explore,\n            true_fn=lambda: action_dist.sampled_action_logp(),\n            false_fn=logp_false_fn)\n\n        return action, logp\n\n    @staticmethod\n    def _get_torch_exploration_action(action_dist, explore):\n        if explore:\n            action = action_dist.sample()\n            logp = action_dist.sampled_action_logp()\n        else:\n            action = action_dist.deterministic_sample()\n            logp = torch.zeros_like(action_dist.sampled_action_logp())\n        return action, logp\n", "path": "rllib/utils/exploration/stochastic_sampling.py"}]}