'Read one file and convert to ids.
Args:
shard_name: file path.
Returns:
list of (id, char_id, global_word_id) tuples.'

def _load_shard(self, shard_name):

    tf.logging.info('Loading    data    from:    %s', shard_name)
    with tf.gfile.Open(shard_name) as f:
        sentences = f.readlines()
    chars_ids = [self.vocab.encode_chars(sentence) for sentence in sentences]
    ids = [self.vocab.encode(sentence) for sentence in sentences]
    global_word_ids = []
    current_idx = 0
    for word_ids in ids:
        current_size = (len(word_ids) - 1)
        cur_ids = np.arange(current_idx, (current_idx + current_size))
        global_word_ids.append(cur_ids)
        current_idx += current_size
    tf.logging.info('Loaded    %d    words.', current_idx)
    tf.logging.info('Finished    loading')
    return zip(ids, chars_ids, global_word_ids)
