Traceback (most recent call last):


 "<stdin>", line 1, in <module>


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/generic.py", line 5691, in astype

**kwargs)


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/internals/managers.py", line 531, in astype

return self.apply('astype', dtype=dtype, **kwargs)


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/internals/managers.py", line 395, in apply

applied = getattr(b, f)(**kwargs)


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/internals/blocks.py", line 534, in astype

**kwargs)


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/internals/blocks.py", line 595, in _astype

dtype = pandas_dtype(dtype)


 "/home/shadchin/t_d/venv/local/lib/python2.7/site-packages/pandas/core/dtypes/common.py", line 2029, in pandas_dtype

raise TypeError("dtype '{}' not understood".format(dtype))
TypeError: dtype '<class 'decimal.Decimal'>' not understood

Traceback (most recent call last):


 "/usr/local/bin/t2t-trainer", line 33, in <module>

tf.app.run()


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 125, in run

_sys.exit(main(argv))


 "/usr/local/bin/t2t-trainer", line 28, in main

t2t_trainer.main(argv)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/bin/t2t_trainer.py", line 387, in main

execute_schedule(exp)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/bin/t2t_trainer.py", line 349, in execute_schedule

getattr(exp, FLAGS.schedule)()


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/trainer_lib.py", line 438, in continuous_train_and_eval

self._eval_spec)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 471, in train_and_evaluate

return executor.run()


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 610, in run

return self.run_local()


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 711, in run_local

saving_listeners=saving_listeners)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2409, in train

rendezvous.raise_errors()


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/error_handling.py", line 128, in raise_errors

six.reraise(typ, value, traceback)


 "/usr/local/lib/python3.6/site-packages/six.py", line 693, in reraise

raise value


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2403, in train

saving_listeners=saving_listeners


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train

loss = self._train_model(input_fn, hooks, saving_listeners)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model

return self._train_model_default(input_fn, hooks, saving_listeners)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1237, in _train_model_default

features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2195, in _call_model_fn

features, labels, mode, config)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1195, in _call_model_fn

model_fn_results = self._model_fn(features=features, **kwargs)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2502, in _model_fn

_train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn))


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2828, in _train_on_tpu_system

device_assignment=ctx.device_assignment)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu.py", line 881, in shard

name=name)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu.py", line 507, in replicate

device_assignment, name)[1]


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu.py", line 684, in split_compile_and_replicate

outputs = computation(*computation_inputs)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 2821, in multi_tpu_train_steps_on_single_shard

single_tpu_train_step, [_INITIAL_LOSS])


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/training_loop.py", line 207, in repeat

cond, body_wrapper, inputs=inputs, infeed_queue=infeed_queue, name=name)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/training_loop.py", line 169, in while_loop

name="")


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 3291, in while_loop

return_same_structure)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 3004, in BuildLoop

pred, body, original_loop_vars, loop_vars, shape_invariants)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2939, in _BuildLoop

body_result = body(*packed_vars_for_body)


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/training_loop.py", line 120, in body_wrapper

outputs = body(*(inputs + dequeue_ops))


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/training_loop.py", line 203, in body_wrapper

return [i + 1] + _convert_to_list(body(*args))


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 1295, in train_step

self._call_model_fn(features, labels))


 "/usr/local/lib/python3.6/site-packages/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py", line 1533, in _call_model_fn

estimator_spec = self._model_fn(features=features, **kwargs)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 1264, in wrapping_model_fn

use_tpu=use_tpu)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 1325, in estimator_model_fn

logits, losses_dict = model(features)
# pylint: disable=not-callable


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/layers/base.py", line 374, in __call__

outputs = super(Layer, self).__call__(inputs, *args, **kwargs)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 757, in __call__

outputs = self.call(inputs, *args, **kwargs)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 233, in call

sharded_logits, losses = self.model_fn_sharded(sharded_features)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 294, in model_fn_sharded

sharded_logits, sharded_losses = dp(self.model_fn, datashard_to_features)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/expert_utils.py", line 231, in __call__

outputs.append(fns[i](*my_args[i], **my_kwargs[i]))


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 320, in model_fn

transformed_features = self.bottom(features)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/utils/t2t_model.py", line 395, in bottom

features[feature_name])


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/layers/modalities.py", line 121, in bottom

return self.bottom_simple(x, "input_emb", reuse=None)


 "/usr/local/lib/python3.6/site-packages/tensor2tensor/layers/modalities.py", line 114, in bottom_simple

ret *= tf.expand_dims(tf.to_float(tf.not_equal(x, 0)), -1)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 866, in binary_op_wrapper

return func(x, y, name=name)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1131, in _mul_dispatch

return gen_math_ops.mul(x, y, name=name)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 5042, in mul

"Mul", x=x, y=y, name=name)


 "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 546, in _apply_op_helper

inferred_from[input_arg.type_attr]))
TypeError: Input 'y' of 'Mul' Op has type float32 that does not match type bfloat16 of argument 'x'.
