import tensorflow as tf

                    worker_device=worker, ps_device='/cpu:0', ps_tasks=1)
            with tf.name_scope('{}_{}'.format(mode, i)) as scope:
                with tf.device(device_setter):
                    net_outputs = self._model(shards[i], mode, **self.config)
                    if mode == Mode.TRAIN:
                        loss = self._loss(net_outputs, shards[i], **self.config)
                        loss += tf.reduce_sum(
                                tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES,
                                                  scope))
                        model_params = tf.trainable_variables()
                        grad = tf.gradients(loss, model_params)
                        tower_losses.append(loss)
                        tower_gradvars.append(zip(grad, model_params))
                        if i == 0:
                            update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
                                                           scope)
                    elif mode == Mode.EVAL:
                        tower_metrics.append(self._metrics(
                            net_outputs, shards[i], **self.config))
                    else:
                        tower_preds.append(net_outputs)

        if mode == Mode.TRAIN:
            return tower_losses, tower_gradvars, update_ops
        elif mode == Mode.EVAL:
            return tower_metrics
        else:
            return tower_preds
