class_docstr: Base class for seq2seq models with embeddings
    
docstr: Model-specific preprocessing for features and labels:
  
  - Creates vocabulary lookup tables for source and target vocab
  - Converts tokens into vocabulary ids
docstr_tok: ["Model-specific","preprocessing","for","features","and","labels",":","-","Creates","vocabulary","lookup","tables","for","source","and","target","vocab","-","Converts","tokens","into","vocabulary","ids"]
code: def _preprocess(self, features, labels):
      """Model-specific preprocessing for features and labels:
  
      - Creates vocabulary lookup tables for source and target vocab
      - Converts tokens into vocabulary ids
      """
  
      # Create vocabulary lookup for source
      (
          source_vocab_to_id,
          source_id_to_vocab,
          source_word_to_count,
          _,
      ) = vocab.create_vocabulary_lookup_table(
          self.source_vocab_info.path
      )
  
      # Create vocabulary look for target
      (
          target_vocab_to_id,
          target_id_to_vocab,
          target_word_to_count,
          _,
      ) = vocab.create_vocabulary_lookup_table(
          self.target_vocab_info.path
      )
  
      # Add vocab tables to graph colection so that we can access them in
      # other places.
      graph_utils.add_dict_to_collection(
          {
              "source_vocab_to_id": source_vocab_to_id,
              "source_id_to_vocab": source_id_to_vocab,
              "source_word_to_count": source_word_to_count,
              "target_vocab_to_id": target_vocab_to_id,
              "target_id_to_vocab": target_id_to_vocab,
              "target_word_to_count": target_word_to_count,
          },
          "vocab_tables",
      )
  
      # Slice source to max_len
      if self.params["source.max_seq_len"] is not None:
          features["source_tokens"] = features["source_tokens"][
              :, : self.params["source.max_seq_len"]
          ]
          features["source_len"] = tf.minimum(
              features["source_len"], self.params["source.max_seq_len"]
          )
  
      # Look up the source ids in the vocabulary
      features["source_ids"] = source_vocab_to_id.lookup(
          features["source_tokens"]
      )
  
      # Maybe reverse the source
      if self.params["source.reverse"] is True:
          features["source_ids"] = tf.reverse_sequence(
              input=features["source_ids"],
              seq_lengths=features["source_len"],
              seq_dim=1,
              batch_dim=0,
              name=None,
          )
  
      features["source_len"] = tf.to_int32(features["source_len"])
      tf.summary.histogram(
          "source_len", tf.to_float(features["source_len"])
      )
  
      if labels is None:
          return features, None
  
      labels = labels.copy()
  
      # Slices targets to max length
      if self.params["target.max_seq_len"] is not None:
          labels["target_tokens"] = labels["target_tokens"][
              :, : self.params["target.max_seq_len"]
          ]
          labels["target_len"] = tf.minimum(
              labels["target_len"], self.params["target.max_seq_len"]
          )
  
      # Look up the target ids in the vocabulary
      labels["target_ids"] = target_vocab_to_id.lookup(
          labels["target_tokens"]
      )
  
      labels["target_len"] = tf.to_int32(labels["target_len"])
      tf.summary.histogram(
          "target_len", tf.to_float(labels["target_len"])
      )
  
      # Keep track of the number of processed tokens
      num_tokens = tf.reduce_sum(labels["target_len"])
      num_tokens += tf.reduce_sum(features["source_len"])
      token_counter_var = tf.Variable(0, "tokens_counter")
      total_tokens = tf.assign_add(token_counter_var, num_tokens)
      tf.summary.scalar("num_tokens", total_tokens)
  
      with tf.control_dependencies([total_tokens]):
          features["source_tokens"] = tf.identity(
              features["source_tokens"]
          )
  
      # Add to graph collection for later use
      graph_utils.add_dict_to_collection(features, "features")
      if labels:
          graph_utils.add_dict_to_collection(labels, "labels")
  
      return features, labels
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  import collections
  import tensorflow
  from seq2seq import graph_utils
  from seq2seq import losses
  from seq2seq.contrib.seq2seq.decoder import _transpose_batch_time
  from seq2seq.data import vocab
  from seq2seq.graph_utils import templatemethod
  from seq2seq.decoders.beam_search_decoder import BeamSearchDecoder
  from seq2seq.inference import beam_search
  from seq2seq.models.model_base import ModelBase, _flatten_dict
label_desc: Use image node_id 12 for calling the Seq2SeqModel obj's underlying member method code with example usage: obj._preprocess(features, labels) and returns: features,  labels, features,  None