prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  policy_gradient 原理介绍部分内容格式存在问题
  https://github.com/PaddlePaddle/models/tree/develop/fluid/policy_gradient 
  policy_gradient  demo介绍部分，看起来格式存在问题，能辛苦调整下吗？或者以什么样的方式可以看到原始的文档呢？ @wanghaoshuang  @lcy-seso 
  
  </issue>
  <code>
  [start of fluid/policy_gradient/brain.py]
  1 import numpy as np
  2 import paddle.v2 as paddle
  3 import paddle.fluid as fluid
  4 # reproducible
  5 np.random.seed(1)
  6 
  7 
  8 class PolicyGradient:
  9     def __init__(
  10             self,
  11             n_actions,
  12             n_features,
  13             learning_rate=0.01,
  14             reward_decay=0.95,
  15             output_graph=False, ):
  16         self.n_actions = n_actions
  17         self.n_features = n_features
  18         self.lr = learning_rate
  19         self.gamma = reward_decay
  20 
  21         self.ep_obs, self.ep_as, self.ep_rs = [], [], []
  22 
  23         self.place = fluid.CPUPlace()
  24         self.exe = fluid.Executor(self.place)
  25 
  26     def build_net(self):
  27 
  28         obs = fluid.layers.data(
  29             name='obs', shape=[self.n_features], dtype='float32')
  30         acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')
  31         vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')
  32         # fc1
  33         fc1 = fluid.layers.fc(
  34             input=obs,
  35             size=10,
  36             act="tanh"  # tanh activation
  37         )
  38         # fc2
  39         self.all_act_prob = fluid.layers.fc(input=fc1,
  40                                             size=self.n_actions,
  41                                             act="softmax")
  42         # to maximize total reward (log_p * R) is to minimize -(log_p * R)
  43         neg_log_prob = fluid.layers.cross_entropy(
  44             input=self.all_act_prob,
  45             label=acts)  # this is negative log of chosen action
  46         neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)
  47         loss = fluid.layers.reduce_mean(
  48             x=neg_log_prob_weight)  # reward guided loss
  49 
  50         sgd_optimizer = fluid.optimizer.SGD(self.lr)
  51         sgd_optimizer.minimize(loss)
  52         self.exe.run(fluid.default_startup_program())
  53 
  54     def choose_action(self, observation):
  55         prob_weights = self.exe.run(
  56             fluid.default_main_program().prune(self.all_act_prob),
  57             feed={"obs": observation[np.newaxis, :]},
  58             fetch_list=[self.all_act_prob])
  59         prob_weights = np.array(prob_weights[0])
  60         action = np.random.choice(
  61             range(prob_weights.shape[1]),
  62             p=prob_weights.ravel())  # select action w.r.t the actions prob
  63         return action
  64 
  65     def store_transition(self, s, a, r):
  66         self.ep_obs.append(s)
  67         self.ep_as.append(a)
  68         self.ep_rs.append(r)
  69 
  70     def learn(self):
  71         # discount and normalize episode reward
  72         discounted_ep_rs_norm = self._discount_and_norm_rewards()
  73         tensor_obs = np.vstack(self.ep_obs).astype("float32")
  74         tensor_as = np.array(self.ep_as).astype("int64")
  75         tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])
  76         tensor_vt = discounted_ep_rs_norm.astype("float32")[:, np.newaxis]
  77         # train on episode
  78         self.exe.run(
  79             fluid.default_main_program(),
  80             feed={
  81                 "obs": tensor_obs,  # shape=[None, n_obs]
  82                 "acts": tensor_as,  # shape=[None, ]
  83                 "vt": tensor_vt  # shape=[None, ]
  84             })
  85         self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data
  86         return discounted_ep_rs_norm
  87 
  88     def _discount_and_norm_rewards(self):
  89         # discount episode rewards
  90         discounted_ep_rs = np.zeros_like(self.ep_rs)
  91         running_add = 0
  92         for t in reversed(range(0, len(self.ep_rs))):
  93             running_add = running_add * self.gamma + self.ep_rs[t]
  94             discounted_ep_rs[t] = running_add
  95 
  96         # normalize episode rewards
  97         discounted_ep_rs -= np.mean(discounted_ep_rs)
  98         discounted_ep_rs /= np.std(discounted_ep_rs)
  99         return discounted_ep_rs
  100
  [end of fluid/policy_gradient/brain.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/fluid/policy_gradient/brain.py b/fluid/policy_gradient/brain.py
  --- a/fluid/policy_gradient/brain.py
  +++ b/fluid/policy_gradient/brain.py
  @@ -45,7 +45,7 @@
               label=acts)  # this is negative log of chosen action
           neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)
           loss = fluid.layers.reduce_mean(
  -            x=neg_log_prob_weight)  # reward guided loss
  +            neg_log_prob_weight)  # reward guided loss
   
           sgd_optimizer = fluid.optimizer.SGD(self.lr)
           sgd_optimizer.minimize(loss)
  
verification_info: {"golden_diff": "diff --git a/fluid/policy_gradient/brain.py b/fluid/policy_gradient/brain.py\n--- a/fluid/policy_gradient/brain.py\n+++ b/fluid/policy_gradient/brain.py\n@@ -45,7 +45,7 @@\n             label=acts)  # this is negative log of chosen action\n         neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)\n         loss = fluid.layers.reduce_mean(\n-            x=neg_log_prob_weight)  # reward guided loss\n+            neg_log_prob_weight)  # reward guided loss\n \n         sgd_optimizer = fluid.optimizer.SGD(self.lr)\n         sgd_optimizer.minimize(loss)\n", "issue": "policy_gradient \u539f\u7406\u4ecb\u7ecd\u90e8\u5206\u5185\u5bb9\u683c\u5f0f\u5b58\u5728\u95ee\u9898\nhttps://github.com/PaddlePaddle/models/tree/develop/fluid/policy_gradient \r\npolicy_gradient  demo\u4ecb\u7ecd\u90e8\u5206\uff0c\u770b\u8d77\u6765\u683c\u5f0f\u5b58\u5728\u95ee\u9898\uff0c\u80fd\u8f9b\u82e6\u8c03\u6574\u4e0b\u5417\uff1f\u6216\u8005\u4ee5\u4ec0\u4e48\u6837\u7684\u65b9\u5f0f\u53ef\u4ee5\u770b\u5230\u539f\u59cb\u7684\u6587\u6863\u5462\uff1f @wanghaoshuang  @lcy-seso \n", "before_files": [{"content": "import numpy as np\nimport paddle.v2 as paddle\nimport paddle.fluid as fluid\n# reproducible\nnp.random.seed(1)\n\n\nclass PolicyGradient:\n    def __init__(\n            self,\n            n_actions,\n            n_features,\n            learning_rate=0.01,\n            reward_decay=0.95,\n            output_graph=False, ):\n        self.n_actions = n_actions\n        self.n_features = n_features\n        self.lr = learning_rate\n        self.gamma = reward_decay\n\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []\n\n        self.place = fluid.CPUPlace()\n        self.exe = fluid.Executor(self.place)\n\n    def build_net(self):\n\n        obs = fluid.layers.data(\n            name='obs', shape=[self.n_features], dtype='float32')\n        acts = fluid.layers.data(name='acts', shape=[1], dtype='int64')\n        vt = fluid.layers.data(name='vt', shape=[1], dtype='float32')\n        # fc1\n        fc1 = fluid.layers.fc(\n            input=obs,\n            size=10,\n            act=\"tanh\"  # tanh activation\n        )\n        # fc2\n        self.all_act_prob = fluid.layers.fc(input=fc1,\n                                            size=self.n_actions,\n                                            act=\"softmax\")\n        # to maximize total reward (log_p * R) is to minimize -(log_p * R)\n        neg_log_prob = fluid.layers.cross_entropy(\n            input=self.all_act_prob,\n            label=acts)  # this is negative log of chosen action\n        neg_log_prob_weight = fluid.layers.elementwise_mul(x=neg_log_prob, y=vt)\n        loss = fluid.layers.reduce_mean(\n            x=neg_log_prob_weight)  # reward guided loss\n\n        sgd_optimizer = fluid.optimizer.SGD(self.lr)\n        sgd_optimizer.minimize(loss)\n        self.exe.run(fluid.default_startup_program())\n\n    def choose_action(self, observation):\n        prob_weights = self.exe.run(\n            fluid.default_main_program().prune(self.all_act_prob),\n            feed={\"obs\": observation[np.newaxis, :]},\n            fetch_list=[self.all_act_prob])\n        prob_weights = np.array(prob_weights[0])\n        action = np.random.choice(\n            range(prob_weights.shape[1]),\n            p=prob_weights.ravel())  # select action w.r.t the actions prob\n        return action\n\n    def store_transition(self, s, a, r):\n        self.ep_obs.append(s)\n        self.ep_as.append(a)\n        self.ep_rs.append(r)\n\n    def learn(self):\n        # discount and normalize episode reward\n        discounted_ep_rs_norm = self._discount_and_norm_rewards()\n        tensor_obs = np.vstack(self.ep_obs).astype(\"float32\")\n        tensor_as = np.array(self.ep_as).astype(\"int64\")\n        tensor_as = tensor_as.reshape([tensor_as.shape[0], 1])\n        tensor_vt = discounted_ep_rs_norm.astype(\"float32\")[:, np.newaxis]\n        # train on episode\n        self.exe.run(\n            fluid.default_main_program(),\n            feed={\n                \"obs\": tensor_obs,  # shape=[None, n_obs]\n                \"acts\": tensor_as,  # shape=[None, ]\n                \"vt\": tensor_vt  # shape=[None, ]\n            })\n        self.ep_obs, self.ep_as, self.ep_rs = [], [], []  # empty episode data\n        return discounted_ep_rs_norm\n\n    def _discount_and_norm_rewards(self):\n        # discount episode rewards\n        discounted_ep_rs = np.zeros_like(self.ep_rs)\n        running_add = 0\n        for t in reversed(range(0, len(self.ep_rs))):\n            running_add = running_add * self.gamma + self.ep_rs[t]\n            discounted_ep_rs[t] = running_add\n\n        # normalize episode rewards\n        discounted_ep_rs -= np.mean(discounted_ep_rs)\n        discounted_ep_rs /= np.std(discounted_ep_rs)\n        return discounted_ep_rs\n", "path": "fluid/policy_gradient/brain.py"}]}