How to impove the speed of tf.data.experimental.CsvDataset in tensorflow 1.13.1?
I am coding a toy example for csv data in Tensorflow. 
I have implemented three types of data loader in tensorflow and pytorch to compare their speeds. Here is the code:

First, with tensorflow api tf.data.experimental.CsvDataset:

def parse_data(x, n_classes):
    x = tf.convert_to_tensor(x)
    return x[:-1], tf.one_hot(indices=tf.cast(x[-1], tf.int32), depth=n_classes)

if __name__=='__main__':
    dataset_train = tf.data.experimental.CsvDataset('/home/david/Dataset/timit/test.csv', [tf.float32] * 430,
                                                    header=False,
                                                    field_delim=' ')
    dataset_train = dataset_train.map(lambda *x_: parse_data(x_, 1928))
    dataset_train = dataset_train.batch(128)
    dataset_train = dataset_train.prefetch(1)
    iterator = dataset_train.make_initializable_iterator()

    x_in, y = iterator.get_next()

    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x_in)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    logits = tf.layers.Dense(units=1928, activation=None)(x)

    loss = tf.losses.softmax_cross_entropy(y, logits)
    optimizer = tf.train.AdamOptimizer()
    optimizer.minimize(loss)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    sess.run(iterator.initializer)
    running_loss = 0.0
    time_last = time.time()
    epoch = 0
    i = 0
    while True:
        try:
            running_loss += sess.run(loss)  # , feed_dict={x: data, y: labels})
            if (i + 1) % 5 == 0:
                print('\r[epoch: %2d, batch: %5d, time: %5f] loss: %.3f' % (
                    epoch + 1, i + 1, time.time() - time_last, running_loss / i), end=' ')
                time_last = time.time()
            i += 1
        except tf.errors.OutOfRangeError:
            pass


Second, using pandas and tf.placeholder:

if __name__ == '__main__':
    x_in = tf.placeholder(shape=[128, 429], dtype=tf.float32)
    y_in = tf.placeholder(shape=[128], dtype=tf.int32)
    y = tf.one_hot(y_in, depth=1928)

    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x_in)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    x = tf.layers.Dense(units=1024, activation=tf.nn.relu)(x)
    logits = tf.layers.Dense(units=1928, activation=None)(x)

    loss = tf.losses.softmax_cross_entropy(y, logits)
    optimizer = tf.train.AdamOptimizer()
    optimizer.minimize(loss)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    w = pd.read_csv('/home/david/Dataset/timit/test.csv', header=None, delim_whitespace=True).values

    for epoch in range(23):
        running_loss = 0.0
        time_last = time.time()
        i = 0
        indexes = np.random.permutation(w.shape[0])
        w_ = w[indexes, :]
        while True:
            if i * 128 + 128 &gt; w.shape[0]:
                break
            running_loss += sess.run(loss,
                                     feed_dict={x_in: w_[i * 128:i * 128 + 128, :-1],
                                                y_in: w_[i * 128:i * 128 + 128, -1]})
            if (i + 1) % 5 == 0:
                print('\r[epoch: %2d, batch: %5d, time: %5f] loss: %.3f' % (
                    epoch + 1, i + 1, time.time() - time_last, running_loss / i), end=' ')
                time_last = time.time()
            i += 1


Third, with pytorch and pandas:

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.fc1 = nn.Linear(429, 1024)
        self.fc2 = nn.Linear(1024, 1024)
        self.fc3 = nn.Linear(1024, 1024)
        self.fc4 = nn.Linear(1024, 1024)
        self.fc5 = nn.Linear(1024, 1928)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = F.relu(self.fc4(x))
        x = self.fc5(x)
        return x


class CsvDataset(data.Dataset):
    """Face Landmarks dataset."""

    def __init__(self, csv_file):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.landmarks_frame = pd.read_csv(csv_file, header=None, delim_whitespace=True)

    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        landmarks = self.landmarks_frame.values[idx, :]
        return landmarks[:-1], landmarks[-1]


if __name__ == '__main__':
    net = Net()
    device = torch.device('cuda:0')
    print(device)
    net.to(device)

    optimizer = optim.Adam(net.parameters(), lr=0.001)
    criterion = nn.CrossEntropyLoss()

    dataset = CsvDataset('/home/david/Dataset/timit/train.csv')
    trainloader = torch.utils.data.DataLoader(dataset, batch_size=128, shuffle=True)

    for epoch in range(23):

        running_loss = 0.0
        time_last = time.time()
        for i, data in enumerate(trainloader):
            inputs, labels = data
            inputs = inputs.float().to(device)
            labels = labels.long().to(device)

            optimizer.zero_grad()

            output = net(inputs)
            loss = criterion(output, labels)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()
            if (i + 1) % 5 == 0:
                print('\r[epoch: %2d, batch: %5d, time: %5f] loss: %.3f' % (
                    epoch + 1, i + 1, time.time() - time_last, running_loss / i), end=' ')
                time_last = time.time()
        print('')
    print('Finished Training')


I record the time cost for training in five batches:


First, CsvDataset: 1.382647s
Second, tf.placeholder: 0.013263s
Third, pytorch: 0.042086s


I guess it is partly because that tf.data.experimental.CsvDataset do io operations before each batch to extract data from csv file (Is this ture, or there are other reasons?). 

However, it is too slow comparing to the other two. Is there any chance for improvement? How can I set the tf.data.experimental.CsvDataset api to load all csv data at the very beginning?

Or can I say that tf.data.experimental.CsvDataset is implemented only for the csv dataset that is too big to store in the memory? Because the time cost seems like intolerable. 

You may play with the batch size in the first example, and if it reads batches from file every time you can prove it if you make it 2x bigger, you may expect 2x speed improvement. I haven't played with (experimental) class CsvDataset  in TF. 

I am sure Pandas reads your document faster and this is part of the reason why you have these times.

Probable the next step you should unset the loss function nn.CrossEntropyLoss(). Most probable have the regression problem and not the classification problem judging by float labels you have at the end.

So try torch.nn.MSELoss as the loss function.
