import tensorflow as tf

        tower_preds = []
        tower_metrics = []
        for i in range(self.n_gpus):
            worker = '/gpu:{}'.format(i)
            device_setter = tf.train.replica_device_setter(
                    worker_device=worker, ps_device='/cpu:0', ps_tasks=1)
            with tf.name_scope('{}_{}'.format(mode, i)) as scope:
                with tf.device(device_setter):
                    net_outputs = self._model(shards[i], mode, **self.config)
                    if mode == Mode.TRAIN:
                        loss = self._loss(net_outputs, shards[i], **self.config)
                        loss += tf.reduce_sum(
                                tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES,
                                                  scope))
