diff --git a/opennmt/decoders/self_attention_decoder.py b/opennmt/decoders/self_attention_decoder.py
index fe1b3a6e..1e40d5d6 100644
--- a/opennmt/decoders/self_attention_decoder.py
+++ b/opennmt/decoders/self_attention_decoder.py
@@ -4,7 +4,7 @@ import tensorflow as tf
 import opennmt.layers.transformer as transformer
 
 from opennmt.decoders.decoder import Decoder, get_embedding_fn
-from opennmt.layers.position import PositionEmbedder
+from opennmt.layers.position import SinusoidalPositionEncoder
 from opennmt.utils.beam_search import beam_search
 
 
@@ -21,7 +21,7 @@ class SelfAttentionDecoder(Decoder):
                dropout=0.1,
                attention_dropout=0.1,
                relu_dropout=0.1,
-               position_encoder=PositionEmbedder()):
+               position_encoder=SinusoidalPositionEncoder()):
     """Initializes the parameters of the decoder.
 
     Args:
diff --git a/opennmt/encoders/self_attention_encoder.py b/opennmt/encoders/self_attention_encoder.py
index 69d299cc..3a942a76 100644
--- a/opennmt/encoders/self_attention_encoder.py
+++ b/opennmt/encoders/self_attention_encoder.py
@@ -4,7 +4,7 @@ import tensorflow as tf
 import opennmt.layers.transformer as transformer
 
 from opennmt.encoders.encoder import Encoder
-from opennmt.layers.position import PositionEmbedder
+from opennmt.layers.position import SinusoidalPositionEncoder
 
 
 class SelfAttentionEncoder(Encoder):
@@ -20,7 +20,7 @@ class SelfAttentionEncoder(Encoder):
                dropout=0.1,
                attention_dropout=0.1,
                relu_dropout=0.1,
-               position_encoder=PositionEmbedder()):
+               position_encoder=SinusoidalPositionEncoder()):
     """Initializes the parameters of the encoder.
 
     Args:
diff --git a/opennmt/layers/__init__.py b/opennmt/layers/__init__.py
index cdf13693..79a97413 100644
--- a/opennmt/layers/__init__.py
+++ b/opennmt/layers/__init__.py
@@ -10,3 +10,4 @@ from opennmt.layers.bridge import ZeroBridge
 from opennmt.layers.bridge import DenseBridge
 
 from opennmt.layers.position import PositionEmbedder
+from opennmt.layers.position import SinusoidalPositionEncoder
diff --git a/opennmt/layers/position.py b/opennmt/layers/position.py
index 358c5007..f344f225 100644
--- a/opennmt/layers/position.py
+++ b/opennmt/layers/position.py
@@ -1,5 +1,6 @@
 """Define position encoder classes."""
 
+import math
 import abc
 import six
 
@@ -96,10 +97,10 @@ class PositionEncoder(object):
     input_dim = inputs.get_shape().as_list()[-1]
 
     position = tf.tile([position], [batch_size])
+    position = tf.expand_dims(position, 1)
 
     with tf.variable_scope("position_encoding"):
       position_encoding = self.encode(position, input_dim, dtype=inputs.dtype)
-      position_encoding = tf.expand_dims(position_encoding, 1)
       return self.reducer.reduce([inputs, position_encoding])
 
   @abc.abstractmethod
@@ -157,3 +158,20 @@ class PositionEmbedder(PositionEncoder):
     embeddings = tf.get_variable(
         "w_embs", shape=[self.maximum_position + 1, depth], dtype=dtype)
     return tf.nn.embedding_lookup(embeddings, positions)
+
+
+class SinusoidalPositionEncoder(PositionEncoder):
+  """Encodes positions with sine waves as described in
+  https://arxiv.org/abs/1706.03762.
+  """
+
+  def encode(self, positions, depth, dtype=tf.float32):
+    batch_size = tf.shape(positions)[0]
+    positions = tf.cast(positions, dtype)
+
+    log_timescale_increment = math.log(10000) / (depth / 2 - 1)
+    inv_timescales = tf.exp(tf.range(depth / 2, dtype=dtype) * -log_timescale_increment)
+    inv_timescales = tf.reshape(tf.tile(inv_timescales, [batch_size]), [batch_size, -1])
+    scaled_time = tf.expand_dims(positions, -1) * tf.expand_dims(inv_timescales, 1)
+
+    return tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=2)
diff --git a/opennmt/models/transformer.py b/opennmt/models/transformer.py
index f095998e..32bdc812 100644
--- a/opennmt/models/transformer.py
+++ b/opennmt/models/transformer.py
@@ -5,7 +5,7 @@ import tensorflow as tf
 from opennmt.models.sequence_to_sequence import SequenceToSequence
 from opennmt.encoders.self_attention_encoder import SelfAttentionEncoder
 from opennmt.decoders.self_attention_decoder import SelfAttentionDecoder
-from opennmt.layers.position import PositionEmbedder
+from opennmt.layers.position import SinusoidalPositionEncoder
 
 
 class Transformer(SequenceToSequence):
@@ -23,7 +23,7 @@ class Transformer(SequenceToSequence):
                dropout=0.1,
                attention_dropout=0.1,
                relu_dropout=0.1,
-               position_encoder=PositionEmbedder(),
+               position_encoder=SinusoidalPositionEncoder(),
                name="transformer"):
     """Initializes a Transformer model.
 

Positional encoding with sinusoids
Implement a new `opennmt.layers.PositionEncoder` with sinusoids as described in "Attention Is All You Need" (https://arxiv.org/abs/1706.03762).
diff --git a/opennmt/tests/position_test.py b/opennmt/tests/position_test.py
index e71ec8c3..7cfb6e45 100644
--- a/opennmt/tests/position_test.py
+++ b/opennmt/tests/position_test.py
@@ -19,6 +19,15 @@ class PositionTest(tf.test.TestCase):
       positions = sess.run(positions)
       self.assertAllEqual([[1, 2, 3, 4, 0, 0, 0], [1, 2, 3, 4, 5, 6, 0]], positions)
 
+  def testSinusoidalPositionEncoder(self):
+    encoder = position.SinusoidalPositionEncoder()
+    positions = position.make_positions([4, 6])
+    depth = 10
+    encoding = encoder.encode(positions, depth)
+    with self.test_session() as sess:
+      encoding = sess.run(encoding)
+      self.assertAllEqual([2, 6, depth], encoding.shape)
+
 
 if __name__ == "__main__":
   tf.test.main()

pip install -e .[TensorFlow]
parse_log_pytest
pytest
nose2 pytest
3.5
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
attrs @ file:///opt/conda/conda-bld/attrs_1642510447205/work
bleach==1.5.0
certifi==2021.5.30
dataclasses==0.8
enum34==1.1.10
html5lib==0.9999999
importlib-metadata @ file:///tmp/build/80754af9/importlib-metadata_1631916693255/work
iniconfig @ file:///home/linux1/recipes/ci/iniconfig_1610983019677/work
Markdown==3.3.7
more-itertools @ file:///tmp/build/80754af9/more-itertools_1637733554872/work
nose2==0.13.0
numpy==1.19.5
-e git+https://github.com/OpenNMT/OpenNMT-tf.git@c46f9b0ce71e23faa0202832ac0cad179bb3bb42#egg=OpenNMT_tf
packaging @ file:///tmp/build/80754af9/packaging_1637314298585/work
pluggy @ file:///tmp/build/80754af9/pluggy_1615976315926/work
protobuf==3.19.6
py @ file:///opt/conda/conda-bld/py_1644396412707/work
pyparsing @ file:///tmp/build/80754af9/pyparsing_1635766073266/work
pytest==6.2.4
PyYAML==6.0.1
six==1.17.0
tensorflow==1.4.0
tensorflow-tensorboard==0.4.0
toml @ file:///tmp/build/80754af9/toml_1616166611790/work
typing_extensions @ file:///opt/conda/conda-bld/typing_extensions_1647553014482/work
Werkzeug==2.0.3
zipp @ file:///tmp/build/80754af9/zipp_1633618647012/work

name: OpenNMT-tf
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - attrs=21.4.0=pyhd3eb1b0_0
  - ca-certificates=2025.2.25=h06a4308_0
  - certifi=2021.5.30=py36h06a4308_0
  - importlib-metadata=4.8.1=py36h06a4308_0
  - importlib_metadata=4.8.1=hd3eb1b0_0
  - iniconfig=1.1.1=pyhd3eb1b0_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.3=he6710b0_2
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - more-itertools=8.12.0=pyhd3eb1b0_0
  - ncurses=6.4=h6a678d5_0
  - openssl=1.1.1w=h7f8727e_0
  - packaging=21.3=pyhd3eb1b0_0
  - pip=21.2.2=py36h06a4308_0
  - pluggy=0.13.1=py36h06a4308_0
  - py=1.11.0=pyhd3eb1b0_0
  - pyparsing=3.0.4=pyhd3eb1b0_0
  - pytest=6.2.4=py36h06a4308_2
  - python=3.6.13=h12debd9_1
  - readline=8.2=h5eee18b_0
  - setuptools=58.0.4=py36h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - toml=0.10.2=pyhd3eb1b0_0
  - typing_extensions=4.1.1=pyh06a4308_0
  - wheel=0.37.1=pyhd3eb1b0_0
  - xz=5.6.4=h5eee18b_1
  - zipp=3.6.0=pyhd3eb1b0_0
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - bleach==1.5.0
      - dataclasses==0.8
      - enum34==1.1.10
      - html5lib==0.9999999
      - markdown==3.3.7
      - nose2==0.13.0
      - numpy==1.19.5
      - protobuf==3.19.6
      - pyyaml==6.0.1
      - six==1.17.0
      - tensorflow==1.4.0
      - tensorflow-tensorboard==0.4.0
      - werkzeug==2.0.3
prefix: /opt/conda/envs/OpenNMT-tf

opennmt/tests/position_test.py::PositionTest::testSinusoidalPositionEncoder
opennmt/tests/position_test.py::PositionTest::testPositionBuilder opennmt/tests/position_test.py::PositionTest::testPositionBuilderWithMaxLen opennmt/tests/position_test.py::PositionTest::test_session
opennmt/layers/__init__.py opennmt/encoders/self_attention_encoder.py opennmt/layers/position.py opennmt/decoders/self_attention_decoder.py opennmt/models/transformer.py
opennmt/layers/__init__.py opennmt/encoders/self_attention_encoder.py opennmt/layers/position.py opennmt/decoders/self_attention_decoder.py opennmt/models/transformer.py