diff --git a/sockeye/training.py b/sockeye/training.py
index <HASH>..<HASH> 100644
--- a/sockeye/training.py
+++ b/sockeye/training.py
@@ -256,8 +256,6 @@ class GluonEarlyStoppingTrainer:
 
         # send sharded inputs to the backend
         for inputs, labels in batch.shards():
-            if self.dtype == C.DTYPE_FP16:
-                inputs = (i.astype(C.DTYPE_FP16, copy=False) for i in inputs)  # type: ignore
             self._parallel.put((inputs, labels))
 
         # get outputs from parallel requests to the backend. Each shard output contains a list of tuples, one for each
diff --git a/sockeye/transformer.py b/sockeye/transformer.py
index <HASH>..<HASH> 100644
--- a/sockeye/transformer.py
+++ b/sockeye/transformer.py
@@ -320,7 +320,7 @@ class TransformerValidLengthMask(mx.gluon.HybridBlock):
         :return:
         """
         # (batch, 1)
-        mask = F.reshape(F.zeros_like(lengths), shape=(-1, 1))
+        mask = F.reshape(F.zeros_like(lengths.astype(self._dtype)), shape=(-1, 1))
         # (batch, seq_len)
         mask = F.broadcast_like(mask, data, lhs_axes=(1,), rhs_axes=(1,))
         # (batch_size, max_length)
Fix FP<I> training: not casting inputs to float<I> due to limited fp<I> range. Requires casting of lengths in transformer valid length mask