prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Poor translation results with the Transformer
  The Transformer model produces very bad translation results. Its implementation should be revised and fixed.
  
  See also the reference implementation at https://github.com/tensorflow/tensor2tensor.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `opennmt/utils/transformer.py`
  Content:
  ```
  1 """Define functions related to the Google's Transformer model."""
  2 
  3 import tensorflow as tf
  4 
  5 
  6 def scaled_dot_attention(queries,
  7                          keys,
  8                          values,
  9                          mode,
  10                          values_length=None,
  11                          mask_future=False,
  12                          dropout=0.0):
  13   """Computes the scaled dot-product attention as described
  14   in https://arxiv.org/abs/1706.03762.
  15 
  16   Args:
  17     queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.
  18     keys: The sequence use to calculate attention scores. A tensor of shape
  19       :math:`[B, T_2, ...]`.
  20     values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.
  21     mode: A ``tf.estimator.ModeKeys`` mode.
  22     values_length: The length of the values to attend.
  23     mask_future: Mask attention to future positions.
  24     dropout: The probability to drop units from the inputs.
  25 
  26   Returns:
  27     A tuple ``(context vector, attention vector)``.
  28   """
  29   # Scaled dot-product between queries and keys.
  30   dot = tf.matmul(queries, keys, transpose_b=True)
  31   dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))
  32 
  33   if values_length is not None:
  34     # Give no weight to illegal connections.
  35     if mask_future:
  36       # When masking the future, a position can only attend to previous timesteps.
  37       mask = tf.map_fn(
  38           lambda x: tf.sequence_mask(
  39               tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),
  40               maxlen=tf.shape(values)[1],
  41               dtype=tf.float32),
  42           values_length,
  43           dtype=tf.float32)
  44     else:
  45       # Otherwise, simply prevent attention on out-of-range positions.
  46       mask = tf.sequence_mask(
  47           values_length,
  48           maxlen=tf.shape(values)[1],
  49           dtype=tf.float32)
  50       mask = tf.expand_dims(mask, axis=1)
  51 
  52     dot = dot * mask + ((1.0 - mask) * tf.float32.min)
  53 
  54   # Compute attention weights.
  55   attn = tf.nn.softmax(dot)
  56   attn = tf.layers.dropout(
  57       attn,
  58       rate=dropout,
  59       training=mode == tf.estimator.ModeKeys.TRAIN)
  60 
  61   # Compute attention context.
  62   context = tf.matmul(attn, values)
  63 
  64   return context, attn
  65 
  66 
  67 def multi_head_attention(num_heads,
  68                          queries,
  69                          keys,
  70                          values,
  71                          mode,
  72                          values_length=None,
  73                          mask_future=False,
  74                          dropout=0.0):
  75   """Computes the multi-head attention as described in
  76   https://arxiv.org/abs/1706.03762.
  77 
  78   Args:
  79     num_heads: The number of attention heads.
  80     queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.
  81     keys: The sequence use to calculate attention scores. A tensor of shape
  82       :math:`[B, T_2, ...]`.
  83     values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.
  84     mode: A ``tf.estimator.ModeKeys`` mode.
  85     values_length: The length of the values to attend.
  86     mask_future: Mask attention to future positions.
  87     dropout: The probability to drop units from the inputs.
  88 
  89   Returns:
  90     The concatenated attention context of each head.
  91   """
  92   input_dim = keys.get_shape().as_list()[-1]
  93 
  94   if input_dim % num_heads != 0:
  95     raise ValueError("Multi head attention requires the input dimension to be a"
  96                      " multiple of {}".format(num_heads))
  97 
  98   head_dim = input_dim / num_heads
  99   heads = []
  100 
  101   for i in range(num_heads):
  102     with tf.variable_scope("head_{}".format(i)):
  103       # Project queries, keys and values to different and smaller subspaces.
  104       queries_proj = tf.layers.conv1d(queries, head_dim, 1)
  105       keys_proj = tf.layers.conv1d(keys, head_dim, 1)
  106       values_proj = tf.layers.conv1d(values, head_dim, 1)
  107 
  108       head_i, _ = scaled_dot_attention(
  109           queries_proj,
  110           keys_proj,
  111           values_proj,
  112           mode,
  113           values_length=values_length,
  114           mask_future=mask_future,
  115           dropout=dropout)
  116 
  117       heads.append(head_i)
  118 
  119   # Concatenate all heads output.
  120   combined = tf.concat(heads, axis=2)
  121   outputs = tf.layers.conv1d(combined, input_dim, 1)
  122 
  123   return outputs
  124 
  125 def feed_forward(x, inner_dim):
  126   """Implements the Transformer's "Feed Forward" layer.
  127 
  128   .. math::
  129 
  130       ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2
  131 
  132   Args:
  133     x: The input.
  134     inner_dim: The number of units of the inner linear transformation.
  135 
  136   Returns:
  137     The transformed input.
  138   """
  139   input_dim = x.get_shape().as_list()[-1]
  140 
  141   inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)
  142   outer = tf.layers.conv1d(inner, input_dim, 1)
  143 
  144   return outer
  145 
  146 def add_and_norm(inputs,
  147                  outputs,
  148                  mode,
  149                  dropout=0.1):
  150   """Implements the Transformer's "Add & Norm" layer.
  151 
  152   Args:
  153     inputs: The input of the previous layer.
  154     outputs: The output of the previous layer.
  155     mode: A ``tf.estimator.ModeKeys`` mode.
  156     dropout: The probability to drop units in :obj:`outputs`.
  157 
  158   Returns:
  159     The residual and normalized output.
  160   """
  161   outputs = tf.layers.dropout(
  162       outputs,
  163       rate=dropout,
  164       training=mode == tf.estimator.ModeKeys.TRAIN)
  165   outputs += inputs
  166   outputs = tf.contrib.layers.layer_norm(outputs)
  167   return outputs
  168
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/opennmt/utils/transformer.py b/opennmt/utils/transformer.py
  --- a/opennmt/utils/transformer.py
  +++ b/opennmt/utils/transformer.py
  @@ -163,5 +163,5 @@
         rate=dropout,
         training=mode == tf.estimator.ModeKeys.TRAIN)
     outputs += inputs
  -  outputs = tf.contrib.layers.layer_norm(outputs)
  +  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)
     return outputs
  
verification_info: {"golden_diff": "diff --git a/opennmt/utils/transformer.py b/opennmt/utils/transformer.py\n--- a/opennmt/utils/transformer.py\n+++ b/opennmt/utils/transformer.py\n@@ -163,5 +163,5 @@\n       rate=dropout,\n       training=mode == tf.estimator.ModeKeys.TRAIN)\n   outputs += inputs\n-  outputs = tf.contrib.layers.layer_norm(outputs)\n+  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)\n   return outputs\n", "issue": "Poor translation results with the Transformer\nThe Transformer model produces very bad translation results. Its implementation should be revised and fixed.\r\n\r\nSee also the reference implementation at https://github.com/tensorflow/tensor2tensor.\n", "before_files": [{"content": "\"\"\"Define functions related to the Google's Transformer model.\"\"\"\n\nimport tensorflow as tf\n\n\ndef scaled_dot_attention(queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the scaled dot-product attention as described\n  in https://arxiv.org/abs/1706.03762.\n\n  Args:\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    A tuple ``(context vector, attention vector)``.\n  \"\"\"\n  # Scaled dot-product between queries and keys.\n  dot = tf.matmul(queries, keys, transpose_b=True)\n  dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))\n\n  if values_length is not None:\n    # Give no weight to illegal connections.\n    if mask_future:\n      # When masking the future, a position can only attend to previous timesteps.\n      mask = tf.map_fn(\n          lambda x: tf.sequence_mask(\n              tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),\n              maxlen=tf.shape(values)[1],\n              dtype=tf.float32),\n          values_length,\n          dtype=tf.float32)\n    else:\n      # Otherwise, simply prevent attention on out-of-range positions.\n      mask = tf.sequence_mask(\n          values_length,\n          maxlen=tf.shape(values)[1],\n          dtype=tf.float32)\n      mask = tf.expand_dims(mask, axis=1)\n\n    dot = dot * mask + ((1.0 - mask) * tf.float32.min)\n\n  # Compute attention weights.\n  attn = tf.nn.softmax(dot)\n  attn = tf.layers.dropout(\n      attn,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n\n  # Compute attention context.\n  context = tf.matmul(attn, values)\n\n  return context, attn\n\n\ndef multi_head_attention(num_heads,\n                         queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the multi-head attention as described in\n  https://arxiv.org/abs/1706.03762.\n\n  Args:\n    num_heads: The number of attention heads.\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    The concatenated attention context of each head.\n  \"\"\"\n  input_dim = keys.get_shape().as_list()[-1]\n\n  if input_dim % num_heads != 0:\n    raise ValueError(\"Multi head attention requires the input dimension to be a\"\n                     \" multiple of {}\".format(num_heads))\n\n  head_dim = input_dim / num_heads\n  heads = []\n\n  for i in range(num_heads):\n    with tf.variable_scope(\"head_{}\".format(i)):\n      # Project queries, keys and values to different and smaller subspaces.\n      queries_proj = tf.layers.conv1d(queries, head_dim, 1)\n      keys_proj = tf.layers.conv1d(keys, head_dim, 1)\n      values_proj = tf.layers.conv1d(values, head_dim, 1)\n\n      head_i, _ = scaled_dot_attention(\n          queries_proj,\n          keys_proj,\n          values_proj,\n          mode,\n          values_length=values_length,\n          mask_future=mask_future,\n          dropout=dropout)\n\n      heads.append(head_i)\n\n  # Concatenate all heads output.\n  combined = tf.concat(heads, axis=2)\n  outputs = tf.layers.conv1d(combined, input_dim, 1)\n\n  return outputs\n\ndef feed_forward(x, inner_dim):\n  \"\"\"Implements the Transformer's \"Feed Forward\" layer.\n\n  .. math::\n\n      ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2\n\n  Args:\n    x: The input.\n    inner_dim: The number of units of the inner linear transformation.\n\n  Returns:\n    The transformed input.\n  \"\"\"\n  input_dim = x.get_shape().as_list()[-1]\n\n  inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)\n  outer = tf.layers.conv1d(inner, input_dim, 1)\n\n  return outer\n\ndef add_and_norm(inputs,\n                 outputs,\n                 mode,\n                 dropout=0.1):\n  \"\"\"Implements the Transformer's \"Add & Norm\" layer.\n\n  Args:\n    inputs: The input of the previous layer.\n    outputs: The output of the previous layer.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    dropout: The probability to drop units in :obj:`outputs`.\n\n  Returns:\n    The residual and normalized output.\n  \"\"\"\n  outputs = tf.layers.dropout(\n      outputs,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n  outputs += inputs\n  outputs = tf.contrib.layers.layer_norm(outputs)\n  return outputs\n", "path": "opennmt/utils/transformer.py"}], "after_files": [{"content": "\"\"\"Define functions related to the Google's Transformer model.\"\"\"\n\nimport tensorflow as tf\n\n\ndef scaled_dot_attention(queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the scaled dot-product attention as described\n  in https://arxiv.org/abs/1706.03762.\n\n  Args:\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    A tuple ``(context vector, attention vector)``.\n  \"\"\"\n  # Scaled dot-product between queries and keys.\n  dot = tf.matmul(queries, keys, transpose_b=True)\n  dot = tf.div(dot, tf.sqrt(tf.cast(tf.shape(keys)[-1], tf.float32)))\n\n  if values_length is not None:\n    # Give no weight to illegal connections.\n    if mask_future:\n      # When masking the future, a position can only attend to previous timesteps.\n      mask = tf.map_fn(\n          lambda x: tf.sequence_mask(\n              tf.minimum(tf.range(tf.shape(values)[1]) + 1, x),\n              maxlen=tf.shape(values)[1],\n              dtype=tf.float32),\n          values_length,\n          dtype=tf.float32)\n    else:\n      # Otherwise, simply prevent attention on out-of-range positions.\n      mask = tf.sequence_mask(\n          values_length,\n          maxlen=tf.shape(values)[1],\n          dtype=tf.float32)\n      mask = tf.expand_dims(mask, axis=1)\n\n    dot = dot * mask + ((1.0 - mask) * tf.float32.min)\n\n  # Compute attention weights.\n  attn = tf.nn.softmax(dot)\n  attn = tf.layers.dropout(\n      attn,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n\n  # Compute attention context.\n  context = tf.matmul(attn, values)\n\n  return context, attn\n\n\ndef multi_head_attention(num_heads,\n                         queries,\n                         keys,\n                         values,\n                         mode,\n                         values_length=None,\n                         mask_future=False,\n                         dropout=0.0):\n  \"\"\"Computes the multi-head attention as described in\n  https://arxiv.org/abs/1706.03762.\n\n  Args:\n    num_heads: The number of attention heads.\n    queries: The sequence of queries. A tensor of shape :math:`[B, T_1, ...]`.\n    keys: The sequence use to calculate attention scores. A tensor of shape\n      :math:`[B, T_2, ...]`.\n    values: The sequence to attend. A tensor of shape :math:`[B, T_2, ...]`.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    values_length: The length of the values to attend.\n    mask_future: Mask attention to future positions.\n    dropout: The probability to drop units from the inputs.\n\n  Returns:\n    The concatenated attention context of each head.\n  \"\"\"\n  input_dim = keys.get_shape().as_list()[-1]\n\n  if input_dim % num_heads != 0:\n    raise ValueError(\"Multi head attention requires the input dimension to be a\"\n                     \" multiple of {}\".format(num_heads))\n\n  head_dim = input_dim / num_heads\n  heads = []\n\n  for i in range(num_heads):\n    with tf.variable_scope(\"head_{}\".format(i)):\n      # Project queries, keys and values to different and smaller subspaces.\n      queries_proj = tf.layers.conv1d(queries, head_dim, 1)\n      keys_proj = tf.layers.conv1d(keys, head_dim, 1)\n      values_proj = tf.layers.conv1d(values, head_dim, 1)\n\n      head_i, _ = scaled_dot_attention(\n          queries_proj,\n          keys_proj,\n          values_proj,\n          mode,\n          values_length=values_length,\n          mask_future=mask_future,\n          dropout=dropout)\n\n      heads.append(head_i)\n\n  # Concatenate all heads output.\n  combined = tf.concat(heads, axis=2)\n  outputs = tf.layers.conv1d(combined, input_dim, 1)\n\n  return outputs\n\ndef feed_forward(x, inner_dim):\n  \"\"\"Implements the Transformer's \"Feed Forward\" layer.\n\n  .. math::\n\n      ffn(x) = max(0, x*W_1 + b_1)*W_2 + b_2\n\n  Args:\n    x: The input.\n    inner_dim: The number of units of the inner linear transformation.\n\n  Returns:\n    The transformed input.\n  \"\"\"\n  input_dim = x.get_shape().as_list()[-1]\n\n  inner = tf.layers.conv1d(x, inner_dim, 1, activation=tf.nn.relu)\n  outer = tf.layers.conv1d(inner, input_dim, 1)\n\n  return outer\n\ndef add_and_norm(inputs,\n                 outputs,\n                 mode,\n                 dropout=0.1):\n  \"\"\"Implements the Transformer's \"Add & Norm\" layer.\n\n  Args:\n    inputs: The input of the previous layer.\n    outputs: The output of the previous layer.\n    mode: A ``tf.estimator.ModeKeys`` mode.\n    dropout: The probability to drop units in :obj:`outputs`.\n\n  Returns:\n    The residual and normalized output.\n  \"\"\"\n  outputs = tf.layers.dropout(\n      outputs,\n      rate=dropout,\n      training=mode == tf.estimator.ModeKeys.TRAIN)\n  outputs += inputs\n  outputs = tf.contrib.layers.layer_norm(outputs, begin_norm_axis=-1)\n  return outputs\n", "path": "opennmt/utils/transformer.py"}]}