'Model-specific preprocessing for features and labels:
- Creates vocabulary lookup tables for source and target vocab
- Converts tokens into vocabulary ids'

def _preprocess(self, features, labels):

    (source_vocab_to_id, source_id_to_vocab, source_word_to_count, _) = vocab.create_vocabulary_lookup_table(self.source_vocab_info.path)
    (target_vocab_to_id, target_id_to_vocab, target_word_to_count, _) = vocab.create_vocabulary_lookup_table(self.target_vocab_info.path)
    graph_utils.add_dict_to_collection({u'source_vocab_to_id': source_vocab_to_id, u'source_id_to_vocab': source_id_to_vocab, u'source_word_to_count': source_word_to_count, u'target_vocab_to_id': target_vocab_to_id, u'target_id_to_vocab': target_id_to_vocab, u'target_word_to_count': target_word_to_count}, u'vocab_tables')
    if (self.params[u'source.max_seq_len'] is not None):
        features[u'source_tokens'] = features[u'source_tokens'][:, :self.params[u'source.max_seq_len']]
        features[u'source_len'] = tf.minimum(features[u'source_len'], self.params[u'source.max_seq_len'])
    features[u'source_ids'] = source_vocab_to_id.lookup(features[u'source_tokens'])
    if (self.params[u'source.reverse'] is True):
        features[u'source_ids'] = tf.reverse_sequence(input=features[u'source_ids'], seq_lengths=features[u'source_len'], seq_dim=1, batch_dim=0, name=None)
    features[u'source_len'] = tf.to_int32(features[u'source_len'])
    tf.summary.histogram(u'source_len', tf.to_float(features[u'source_len']))
    if (labels is None):
        return (features, None)
    labels = labels.copy()
    if (self.params[u'target.max_seq_len'] is not None):
        labels[u'target_tokens'] = labels[u'target_tokens'][:, :self.params[u'target.max_seq_len']]
        labels[u'target_len'] = tf.minimum(labels[u'target_len'], self.params[u'target.max_seq_len'])
    labels[u'target_ids'] = target_vocab_to_id.lookup(labels[u'target_tokens'])
    labels[u'target_len'] = tf.to_int32(labels[u'target_len'])
    tf.summary.histogram(u'target_len', tf.to_float(labels[u'target_len']))
    num_tokens = tf.reduce_sum(labels[u'target_len'])
    num_tokens += tf.reduce_sum(features[u'source_len'])
    token_counter_var = tf.Variable(0, u'tokens_counter')
    total_tokens = tf.assign_add(token_counter_var, num_tokens)
    tf.summary.scalar(u'num_tokens', total_tokens)
    with tf.control_dependencies([total_tokens]):
        features[u'source_tokens'] = tf.identity(features[u'source_tokens'])
    graph_utils.add_dict_to_collection(features, u'features')
    if labels:
        graph_utils.add_dict_to_collection(labels, u'labels')
    return (features, labels)
