prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  PaddleRL policy_gradient Typo
  default_main_program误写为defaul_main_program
  all_act_prob 未被声明为成员变量
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `legacy/PaddleRL/policy_gradient/brain.py`
  Content:
  ```
  1 import numpy as np
  2 import paddle.fluid as fluid
  3 # reproducible
  4 np.random.seed(1)
  5 
  6 
  7 class PolicyGradient:
  8     def __init__(
  9             self,
  10             n_actions,
  11             n_features,
  12             learning_rate=0.01,
  13             reward_decay=0.95,
  14             output_graph=False, ):
  15         self.n_actions = n_actions
  16         self.n_features = n_features
  17         self.lr = learning_rate
  18         self.gamma = reward_decay
  19 
  20         self.ep_obs, self.ep_as, self.ep_rs = [], [], []
  21 
  22         self.place = fluid.CPUPlace()
  23         self.exe = fluid.Executor(self.place)
  24 
  25     def build_net(self):
  26 
  27         obs = fluid.layers.data(
  28             name='obs', shape=[self.n_features], dtype='float32')
  29         acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')
  30         vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')
  31         # fc1
  32         fc1 = fluid.layers.fc(input=obs, size=10, act="tanh")  # tanh activation
  33         # fc2
  34         all_act_prob = fluid.layers.fc(input=fc1,
  35                                        size=self.n_actions,
  36                                        act="softmax")
  37         self.inferece_program = fluid.defaul_main_program().clone()
  38         # to maximize total reward (log_p * R) is to minimize -(log_p * R)
  39         neg_log_prob = fluid.layers.cross_entropy(
  40             input=self.all_act_prob,
  41             label=acts)  # this is negative log of chosen action
  42         neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)
  43         loss = fluid.layers.reduce_mean(
  44             neg_log_prob_weight)  # reward guided loss
  45 
  46         sgd_optimizer = fluid.optimizer.SGD(self.lr)
  47         sgd_optimizer.minimize(loss)
  48         self.exe.run(fluid.default_startup_program())
  49 
  50     def choose_action(self, observation):
  51         prob_weights = self.exe.run(self.inferece_program,
  52                                     feed={"obs": observation[np.newaxis, :]},
  53                                     fetch_list=[self.all_act_prob])
  54         prob_weights = np.array(prob_weights[0])
  55         # select action w.r.t the actions prob
  56         action = np.random.choice(
  57             range(prob_weights.shape[1]), p=prob_weights.ravel())
  58         return action
  59 
  60     def store_transition(self, s, a, r):
  61         self.ep_obs.append(s)
  62         self.ep_as.append(a)
  63         self.ep_rs.append(r)
  64 
  65     def learn(self):
  66         # discount and normalize episode reward
  67         discounted_ep_rs_norm = self._discount_and_norm_rewards()
  68         tensor_obs = np.vstack(self.ep_obs).astype("float32")
  69         tensor_as = np.array(self.ep_as).astype("int64")
  70         tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])
  71         tensor_vt = discounted_ep_rs_norm.astype("float32")[:, np.newaxis]
  72         # train on episode
  73         self.exe.run(
  74             fluid.default_main_program(),
  75             feed={
  76                 "obs": tensor_obs,  # shape=[None, n_obs]
  77                 "acts": tensor_as,  # shape=[None, ]
  78                 "vt": tensor_vt  # shape=[None, ]
  79             })
  80         self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data
  81         return discounted_ep_rs_norm
  82 
  83     def _discount_and_norm_rewards(self):
  84         # discount episode rewards
  85         discounted_ep_rs = np.zeros_like(self.ep_rs)
  86         running_add = 0
  87         for t in reversed(range(0, len(self.ep_rs))):
  88             running_add = running_add * self.gamma + self.ep_rs[t]
  89             discounted_ep_rs[t] = running_add
  90 
  91         # normalize episode rewards
  92         discounted_ep_rs -= np.mean(discounted_ep_rs)
  93         discounted_ep_rs /= np.std(discounted_ep_rs)
  94         return discounted_ep_rs
  95
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/legacy/PaddleRL/policy_gradient/brain.py b/legacy/PaddleRL/policy_gradient/brain.py
  --- a/legacy/PaddleRL/policy_gradient/brain.py
  +++ b/legacy/PaddleRL/policy_gradient/brain.py
  @@ -31,10 +31,10 @@
           # fc1
           fc1 = fluid.layers.fc(input=obs, size=10, act="tanh")  # tanh activation
           # fc2
  -        all_act_prob = fluid.layers.fc(input=fc1,
  +        self.all_act_prob = fluid.layers.fc(input=fc1,
                                          size=self.n_actions,
                                          act="softmax")
  -        self.inferece_program = fluid.defaul_main_program().clone()
  +        self.inferece_program = fluid.default_main_program().clone()
           # to maximize total reward (log_p * R) is to minimize -(log_p * R)
           neg_log_prob = fluid.layers.cross_entropy(
               input=self.all_act_prob,
  
verification_info: {"golden_diff": "diff --git a/legacy/PaddleRL/policy_gradient/brain.py b/legacy/PaddleRL/policy_gradient/brain.py\n--- a/legacy/PaddleRL/policy_gradient/brain.py\n+++ b/legacy/PaddleRL/policy_gradient/brain.py\n@@ -31,10 +31,10 @@\n         # fc1\n         fc1 = fluid.layers.fc(input=obs, size=10, act=\"tanh\")  # tanh activation\n         # fc2\n-        all_act_prob = fluid.layers.fc(input=fc1,\n+        self.all_act_prob = fluid.layers.fc(input=fc1,\n                                        size=self.n_actions,\n                                        act=\"softmax\")\n-        self.inferece_program = fluid.defaul_main_program().clone()\n+        self.inferece_program = fluid.default_main_program().clone()\n         # to maximize total reward (log_p * R) is to minimize -(log_p * R)\n         neg_log_prob = fluid.layers.cross_entropy(\n             input=self.all_act_prob,\n", "issue": "PaddleRL policy_gradient Typo\ndefault_main_program\u8bef\u5199\u4e3adefaul_main_program\r\nall_act_prob \u672a\u88ab\u58f0\u660e\u4e3a\u6210\u5458\u53d8\u91cf\n", "before_files": [{"content": "import numpy as np\nimport paddle.fluid as fluid\n# reproducible\nnp.random.seed(1)\n\n\nclass PolicyGradient:\n    def __init__(\n            self,\n            n_actions,\n            n_features,\n            learning_rate=0.01,\n            reward_decay=0.95,\n            output_graph=False, ):\n        self.n_actions = n_actions\n        self.n_features = n_features\n        self.lr = learning_rate\n        self.gamma = reward_decay\n\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []\n\n        self.place = fluid.CPUPlace()\n        self.exe = fluid.Executor(self.place)\n\n    def build_net(self):\n\n        obs = fluid.layers.data(\n            name='obs', shape=[self.n_features], dtype='float32')\n        acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')\n        vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')\n        # fc1\n        fc1 = fluid.layers.fc(input=obs, size=10, act=\"tanh\")  # tanh activation\n        # fc2\n        all_act_prob = fluid.layers.fc(input=fc1,\n                                       size=self.n_actions,\n                                       act=\"softmax\")\n        self.inferece_program = fluid.defaul_main_program().clone()\n        # to maximize total reward (log_p * R) is to minimize -(log_p * R)\n        neg_log_prob = fluid.layers.cross_entropy(\n            input=self.all_act_prob,\n            label=acts)  # this is negative log of chosen action\n        neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)\n        loss = fluid.layers.reduce_mean(\n            neg_log_prob_weight)  # reward guided loss\n\n        sgd_optimizer = fluid.optimizer.SGD(self.lr)\n        sgd_optimizer.minimize(loss)\n        self.exe.run(fluid.default_startup_program())\n\n    def choose_action(self, observation):\n        prob_weights = self.exe.run(self.inferece_program,\n                                    feed={\"obs\": observation[np.newaxis, :]},\n                                    fetch_list=[self.all_act_prob])\n        prob_weights = np.array(prob_weights[0])\n        # select action w.r.t the actions prob\n        action = np.random.choice(\n            range(prob_weights.shape[1]), p=prob_weights.ravel())\n        return action\n\n    def store_transition(self, s, a, r):\n        self.ep_obs.append(s)\n        self.ep_as.append(a)\n        self.ep_rs.append(r)\n\n    def learn(self):\n        # discount and normalize episode reward\n        discounted_ep_rs_norm = self._discount_and_norm_rewards()\n        tensor_obs = np.vstack(self.ep_obs).astype(\"float32\")\n        tensor_as = np.array(self.ep_as).astype(\"int64\")\n        tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])\n        tensor_vt = discounted_ep_rs_norm.astype(\"float32\")[:, np.newaxis]\n        # train on episode\n        self.exe.run(\n            fluid.default_main_program(),\n            feed={\n                \"obs\": tensor_obs,  # shape=[None, n_obs]\n                \"acts\": tensor_as,  # shape=[None, ]\n                \"vt\": tensor_vt  # shape=[None, ]\n            })\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data\n        return discounted_ep_rs_norm\n\n    def _discount_and_norm_rewards(self):\n        # discount episode rewards\n        discounted_ep_rs = np.zeros_like(self.ep_rs)\n        running_add = 0\n        for t in reversed(range(0, len(self.ep_rs))):\n            running_add = running_add * self.gamma + self.ep_rs[t]\n            discounted_ep_rs[t] = running_add\n\n        # normalize episode rewards\n        discounted_ep_rs -= np.mean(discounted_ep_rs)\n        discounted_ep_rs /= np.std(discounted_ep_rs)\n        return discounted_ep_rs\n", "path": "legacy/PaddleRL/policy_gradient/brain.py"}], "after_files": [{"content": "import numpy as np\nimport paddle.fluid as fluid\n# reproducible\nnp.random.seed(1)\n\n\nclass PolicyGradient:\n    def __init__(\n            self,\n            n_actions,\n            n_features,\n            learning_rate=0.01,\n            reward_decay=0.95,\n            output_graph=False, ):\n        self.n_actions = n_actions\n        self.n_features = n_features\n        self.lr = learning_rate\n        self.gamma = reward_decay\n\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []\n\n        self.place = fluid.CPUPlace()\n        self.exe = fluid.Executor(self.place)\n\n    def build_net(self):\n\n        obs = fluid.layers.data(\n            name='obs', shape=[self.n_features], dtype='float32')\n        acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')\n        vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')\n        # fc1\n        fc1 = fluid.layers.fc(input=obs, size=10, act=\"tanh\")  # tanh activation\n        # fc2\n        self.all_act_prob = fluid.layers.fc(input=fc1,\n                                       size=self.n_actions,\n                                       act=\"softmax\")\n        self.inferece_program = fluid.default_main_program().clone()\n        # to maximize total reward (log_p * R) is to minimize -(log_p * R)\n        neg_log_prob = fluid.layers.cross_entropy(\n            input=self.all_act_prob,\n            label=acts)  # this is negative log of chosen action\n        neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)\n        loss = fluid.layers.reduce_mean(\n            neg_log_prob_weight)  # reward guided loss\n\n        sgd_optimizer = fluid.optimizer.SGD(self.lr)\n        sgd_optimizer.minimize(loss)\n        self.exe.run(fluid.default_startup_program())\n\n    def choose_action(self, observation):\n        prob_weights = self.exe.run(self.inferece_program,\n                                    feed={\"obs\": observation[np.newaxis, :]},\n                                    fetch_list=[self.all_act_prob])\n        prob_weights = np.array(prob_weights[0])\n        # select action w.r.t the actions prob\n        action = np.random.choice(\n            range(prob_weights.shape[1]), p=prob_weights.ravel())\n        return action\n\n    def store_transition(self, s, a, r):\n        self.ep_obs.append(s)\n        self.ep_as.append(a)\n        self.ep_rs.append(r)\n\n    def learn(self):\n        # discount and normalize episode reward\n        discounted_ep_rs_norm = self._discount_and_norm_rewards()\n        tensor_obs = np.vstack(self.ep_obs).astype(\"float32\")\n        tensor_as = np.array(self.ep_as).astype(\"int64\")\n        tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])\n        tensor_vt = discounted_ep_rs_norm.astype(\"float32\")[:, np.newaxis]\n        # train on episode\n        self.exe.run(\n            fluid.default_main_program(),\n            feed={\n                \"obs\": tensor_obs,  # shape=[None, n_obs]\n                \"acts\": tensor_as,  # shape=[None, ]\n                \"vt\": tensor_vt  # shape=[None, ]\n            })\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data\n        return discounted_ep_rs_norm\n\n    def _discount_and_norm_rewards(self):\n        # discount episode rewards\n        discounted_ep_rs = np.zeros_like(self.ep_rs)\n        running_add = 0\n        for t in reversed(range(0, len(self.ep_rs))):\n            running_add = running_add * self.gamma + self.ep_rs[t]\n            discounted_ep_rs[t] = running_add\n\n        # normalize episode rewards\n        discounted_ep_rs -= np.mean(discounted_ep_rs)\n        discounted_ep_rs /= np.std(discounted_ep_rs)\n        return discounted_ep_rs\n", "path": "legacy/PaddleRL/policy_gradient/brain.py"}]}