nmt/model.py
def _build_decoder(self, encoder_outputs, encoder_state, hparams):
    """Build and run a RNN decoder with a final projection layer.

    Args:
      encoder_outputs: The outputs of encoder for every time step.
      encoder_state: The final state of the encoder.
      hparams: The Hyperparameters configurations.

    Returns:
      A tuple of final logits and final decoder state:
        logits: size [time, batch_size, vocab_size] when time_major=True.
    """
    tgt_sos_id = tf.cast(self.tgt_vocab_table.lookup(tf.constant(hparams.sos)),
                         tf.int32)
    tgt_eos_id = tf.cast(self.tgt_vocab_table.lookup(tf.constant(hparams.eos)),
                         tf.int32)
    iterator = self.iterator

    # maximum_iteration: The maximum decoding steps.
    maximum_iterations = self._get_infer_maximum_iterations(hparams)

    ## Decoder.
    with tf.variable_scope("decoder") as decoder_scope:
      cell, decoder_initial_state = self._build_decoder_cell(
          hparams, encoder_outputs, encoder_state,
          iterator.source_spectrum_length)
      
      # self.init_state_dec = decoder_initial_state
      #decoder_initial_state = tf.transpose(decoder_initial_state)
      ## Train or eval
      if self.mode != tf.contrib.learn.ModeKeys.INFER:
        # decoder_emp_inp: [max_time, batch_size, num_units]
        target_input = iterator.target_input
        aa_spec = iterator.aa_spectrum
        if self.time_major:
          target_input = tf.transpose(target_input)
          aa_spec = tf.transpose(aa_spec, perm=[1,0,2,3])
        decoder_emb_inp = tf.nn.embedding_lookup(
            self.embedding_decoder, target_input)
        
        # reshape input from [time, batch_size, 160, full_input_window] to [time * batch_size, 160, full_input_window]
        spec_shape = tf.shape(aa_spec)
        aa_spec = tf.reshape(aa_spec, tf.concat(([spec_shape[0] * spec_shape[1]], spec_shape[2:]), 0))
        interested_regions_num = 8 * aa_conf.vocab_size # + aa_conf.vocab_size
        aa_spec.set_shape([None, interested_regions_num, aa_conf.full_aa_window])
        
        # apply cnn to candidate input, this layer produce output of size
        decoder_aa_spec_inp = self.input_cnn_1(aa_spec) # [time * batch_size, filters=32, full_window_size]
        decoder_aa_spec_inp = self.input_cnn_2(decoder_aa_spec_inp) # [time * batch_size, filters=32, full_window_size]
  
        decoder_aa_spec_inp = tf.contrib.layers.flatten(decoder_aa_spec_inp) # [time * batch_size, filters * full_window_size]
        aa_feature_size = tf.shape(decoder_aa_spec_inp)[1]
        
        # reshape back to [time, batch_size, filters=32]
        decoder_aa_spec_inp = tf.reshape(decoder_aa_spec_inp, [spec_shape[0], spec_shape[1], aa_feature_size])
        decoder_aa_spec_inp.set_shape([None, None, interested_regions_num * 64]) # 32 * aa_conf.full_aa_window])
        
        # apply dense layer -> [time, batch_size, 512]
        decoder_aa_spec_inp = self.input_dense_1(decoder_aa_spec_inp)
        decoder_aa_spec_inp = self.input_dense_2(decoder_aa_spec_inp)

        combine_inp = tf.concat([decoder_emb_inp,decoder_aa_spec_inp], 2)
        
        # Helper
        helper = tf.contrib.seq2seq.TrainingHelper(
            # decoder_emb_inp, iterator.target_sequence_length,
            combine_inp, iterator.target_sequence_length,
            time_major=self.time_major)

        # Decoder
        my_decoder = tf.contrib.seq2seq.BasicDecoder(
            cell,
            helper,
            decoder_initial_state,)

        # Dynamic decoding
        outputs, final_context_state, _ = tf.contrib.seq2seq.dynamic_decode(
            my_decoder,
            output_time_major=self.time_major,
            swap_memory=True,
            scope=decoder_scope)

        sample_id = outputs.sample_id
        step_log_probs = tf.no_op()
        logits = self.output_layer(outputs.rnn_output)

      ## Inference
      else:
        beam_width = hparams.beam_width
        length_penalty_weight = hparams.length_penalty_weight
        start_tokens = tf.fill([self.batch_size], tgt_sos_id)
        end_token = tgt_eos_id

        if beam_width > 0:
          peptide_mass = iterator.peptide_mass
          my_decoder = ion_beam_search_decoder.IonBeamSearchDecoder(
              cell=cell,
              embedding=self.embedding_decoder,
              start_tokens=start_tokens,
              end_token=end_token,
              initial_state=decoder_initial_state,
              beam_width=beam_width,
              source_spectrum=iterator.source,
              peptide_mass=peptide_mass,
              suffix_dp_table=self.suffix_dp_table,
              aa_weight_table=self.aa_weight_table,
              decoder_input_model_fn=self.decoder_input_model_fn,
              output_layer=self.output_layer,
              length_penalty_weight=length_penalty_weight)
        else:
          # Helper
          sampling_temperature = hparams.sampling_temperature
          if sampling_temperature > 0.0:
            helper = tf.contrib.seq2seq.SampleEmbeddingHelper(
                self.embedding_decoder, start_tokens, end_token,
                softmax_temperature=sampling_temperature,
                seed=hparams.random_seed)
          else:
            helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(
                self.embedding_decoder, start_tokens, end_token)

          # Decoder
          my_decoder = tf.contrib.seq2seq.BasicDecoder(
              cell,
              helper,
              decoder_initial_state,
              output_layer=self.output_layer  # applied per timestep
          )

        # Dynamic decoding
        outputs, final_context_state, _ = tf.contrib.seq2seq.dynamic_decode(
            my_decoder,
            maximum_iterations=maximum_iterations,
            output_time_major=self.time_major,
            swap_memory=True,
            scope=decoder_scope)

        if beam_width > 0:
          logits = tf.no_op()
          step_log_probs = outputs.step_log_probs
          sample_id = outputs.predicted_ids
        else:
          logits = outputs.rnn_output
          step_log_probs = tf.no_op()
          sample_id = outputs.sample_id

    return logits, sample_id, final_context_state, step_log_probs
Build and run a RNN decoder with a final projection layer.

    Args:
      encoder_outputs: The outputs of encoder for every time step.
      encoder_state: The final state of the encoder.
      hparams: The Hyperparameters configurations.

    Returns:
      A tuple of final logits and final decoder state:
        logits: size [time, batch_size, vocab_size] when time_major=True.
    
Build and run a RNN decoder with a final projection layer.
Build and run a RNN decoder with a final projection layer .
def _build_decoder(self, encoder_outputs, encoder_state, hparams):
    tgt_sos_id = tf.cast(self.tgt_vocab_table.lookup(tf.constant(hparams.sos)),
                         tf.int32)
    tgt_eos_id = tf.cast(self.tgt_vocab_table.lookup(tf.constant(hparams.eos)),
                         tf.int32)
    iterator = self.iterator
    maximum_iterations = self._get_infer_maximum_iterations(hparams)
    with tf.variable_scope("decoder") as decoder_scope:
      cell, decoder_initial_state = self._build_decoder_cell(
          hparams, encoder_outputs, encoder_state,
          iterator.source_spectrum_length)
      if self.mode != tf.contrib.learn.ModeKeys.INFER:
        target_input = iterator.target_input
        aa_spec = iterator.aa_spectrum
        if self.time_major:
          target_input = tf.transpose(target_input)
          aa_spec = tf.transpose(aa_spec, perm=[1,0,2,3])
        decoder_emb_inp = tf.nn.embedding_lookup(
            self.embedding_decoder, target_input)
        spec_shape = tf.shape(aa_spec)
        aa_spec = tf.reshape(aa_spec, tf.concat(([spec_shape[0] * spec_shape[1]], spec_shape[2:]), 0))
        interested_regions_num = 8 * aa_conf.vocab_size 
        aa_spec.set_shape([None, interested_regions_num, aa_conf.full_aa_window])
        decoder_aa_spec_inp = self.input_cnn_1(aa_spec) 
        decoder_aa_spec_inp = self.input_cnn_2(decoder_aa_spec_inp) 
        decoder_aa_spec_inp = tf.contrib.layers.flatten(decoder_aa_spec_inp) 
        aa_feature_size = tf.shape(decoder_aa_spec_inp)[1]
        decoder_aa_spec_inp = tf.reshape(decoder_aa_spec_inp, [spec_shape[0], spec_shape[1], aa_feature_size])
        decoder_aa_spec_inp.set_shape([None, None, interested_regions_num * 64]) 
        decoder_aa_spec_inp = self.input_dense_1(decoder_aa_spec_inp)
        decoder_aa_spec_inp = self.input_dense_2(decoder_aa_spec_inp)
        combine_inp = tf.concat([decoder_emb_inp,decoder_aa_spec_inp], 2)
        helper = tf.contrib.seq2seq.TrainingHelper(
            combine_inp, iterator.target_sequence_length,
            time_major=self.time_major)
        my_decoder = tf.contrib.seq2seq.BasicDecoder(
            cell,
            helper,
            decoder_initial_state,)
        outputs, final_context_state, _ = tf.contrib.seq2seq.dynamic_decode(
            my_decoder,
            output_time_major=self.time_major,
            swap_memory=True,
            scope=decoder_scope)
        sample_id = outputs.sample_id
        step_log_probs = tf.no_op()
        logits = self.output_layer(outputs.rnn_output)
      else:
        beam_width = hparams.beam_width
        length_penalty_weight = hparams.length_penalty_weight
        start_tokens = tf.fill([self.batch_size], tgt_sos_id)
        end_token = tgt_eos_id
        if beam_width > 0:
          peptide_mass = iterator.peptide_mass
          my_decoder = ion_beam_search_decoder.IonBeamSearchDecoder(
              cell=cell,
              embedding=self.embedding_decoder,
              start_tokens=start_tokens,
              end_token=end_token,
              initial_state=decoder_initial_state,
              beam_width=beam_width,
              source_spectrum=iterator.source,
              peptide_mass=peptide_mass,
              suffix_dp_table=self.suffix_dp_table,
              aa_weight_table=self.aa_weight_table,
              decoder_input_model_fn=self.decoder_input_model_fn,
              output_layer=self.output_layer,
              length_penalty_weight=length_penalty_weight)
        else:
          sampling_temperature = hparams.sampling_temperature
          if sampling_temperature > 0.0:
            helper = tf.contrib.seq2seq.SampleEmbeddingHelper(
                self.embedding_decoder, start_tokens, end_token,
                softmax_temperature=sampling_temperature,
                seed=hparams.random_seed)
          else:
            helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(
                self.embedding_decoder, start_tokens, end_token)
          my_decoder = tf.contrib.seq2seq.BasicDecoder(
              cell,
              helper,
              decoder_initial_state,
              output_layer=self.output_layer  
          )
        outputs, final_context_state, _ = tf.contrib.seq2seq.dynamic_decode(
            my_decoder,
            maximum_iterations=maximum_iterations,
            output_time_major=self.time_major,
            swap_memory=True,
            scope=decoder_scope)
        if beam_width > 0:
          logits = tf.no_op()
          step_log_probs = outputs.step_log_probs
          sample_id = outputs.predicted_ids
        else:
          logits = outputs.rnn_output
          step_log_probs = tf.no_op()
          sample_id = outputs.sample_id
    return logits, sample_id, final_context_state, step_log_probs
def _build_decoder ( self , encoder_outputs , encoder_state , hparams ) : tgt_sos_id = tf . cast ( self . tgt_vocab_table . lookup ( tf . constant ( hparams . sos ) ) , tf . int32 ) tgt_eos_id = tf . cast ( self . tgt_vocab_table . lookup ( tf . constant ( hparams . eos ) ) , tf . int32 ) iterator = self . iterator maximum_iterations = self . _get_infer_maximum_iterations ( hparams ) with tf . variable_scope ( "decoder" ) as decoder_scope : cell , decoder_initial_state = self . _build_decoder_cell ( hparams , encoder_outputs , encoder_state , iterator . source_spectrum_length ) if self . mode != tf . contrib . learn . ModeKeys . INFER : target_input = iterator . target_input aa_spec = iterator . aa_spectrum if self . time_major : target_input = tf . transpose ( target_input ) aa_spec = tf . transpose ( aa_spec , perm = [ 1 , 0 , 2 , 3 ] ) decoder_emb_inp = tf . nn . embedding_lookup ( self . embedding_decoder , target_input ) spec_shape = tf . shape ( aa_spec ) aa_spec = tf . reshape ( aa_spec , tf . concat ( ( [ spec_shape [ 0 ] * spec_shape [ 1 ] ] , spec_shape [ 2 : ] ) , 0 ) ) interested_regions_num = 8 * aa_conf . vocab_size aa_spec . set_shape ( [ None , interested_regions_num , aa_conf . full_aa_window ] ) decoder_aa_spec_inp = self . input_cnn_1 ( aa_spec ) decoder_aa_spec_inp = self . input_cnn_2 ( decoder_aa_spec_inp ) decoder_aa_spec_inp = tf . contrib . layers . flatten ( decoder_aa_spec_inp ) aa_feature_size = tf . shape ( decoder_aa_spec_inp ) [ 1 ] decoder_aa_spec_inp = tf . reshape ( decoder_aa_spec_inp , [ spec_shape [ 0 ] , spec_shape [ 1 ] , aa_feature_size ] ) decoder_aa_spec_inp . set_shape ( [ None , None , interested_regions_num * 64 ] ) decoder_aa_spec_inp = self . input_dense_1 ( decoder_aa_spec_inp ) decoder_aa_spec_inp = self . input_dense_2 ( decoder_aa_spec_inp ) combine_inp = tf . concat ( [ decoder_emb_inp , decoder_aa_spec_inp ] , 2 ) helper = tf . contrib . seq2seq . TrainingHelper ( combine_inp , iterator . target_sequence_length , time_major = self . time_major ) my_decoder = tf . contrib . seq2seq . BasicDecoder ( cell , helper , decoder_initial_state , ) outputs , final_context_state , _ = tf . contrib . seq2seq . dynamic_decode ( my_decoder , output_time_major = self . time_major , swap_memory = True , scope = decoder_scope ) sample_id = outputs . sample_id step_log_probs = tf . no_op ( ) logits = self . output_layer ( outputs . rnn_output ) else : beam_width = hparams . beam_width length_penalty_weight = hparams . length_penalty_weight start_tokens = tf . fill ( [ self . batch_size ] , tgt_sos_id ) end_token = tgt_eos_id if beam_width > 0 : peptide_mass = iterator . peptide_mass my_decoder = ion_beam_search_decoder . IonBeamSearchDecoder ( cell = cell , embedding = self . embedding_decoder , start_tokens = start_tokens , end_token = end_token , initial_state = decoder_initial_state , beam_width = beam_width , source_spectrum = iterator . source , peptide_mass = peptide_mass , suffix_dp_table = self . suffix_dp_table , aa_weight_table = self . aa_weight_table , decoder_input_model_fn = self . decoder_input_model_fn , output_layer = self . output_layer , length_penalty_weight = length_penalty_weight ) else : sampling_temperature = hparams . sampling_temperature if sampling_temperature > 0.0 : helper = tf . contrib . seq2seq . SampleEmbeddingHelper ( self . embedding_decoder , start_tokens , end_token , softmax_temperature = sampling_temperature , seed = hparams . random_seed ) else : helper = tf . contrib . seq2seq . GreedyEmbeddingHelper ( self . embedding_decoder , start_tokens , end_token ) my_decoder = tf . contrib . seq2seq . BasicDecoder ( cell , helper , decoder_initial_state , output_layer = self . output_layer ) outputs , final_context_state , _ = tf . contrib . seq2seq . dynamic_decode ( my_decoder , maximum_iterations = maximum_iterations , output_time_major = self . time_major , swap_memory = True , scope = decoder_scope ) if beam_width > 0 : logits = tf . no_op ( ) step_log_probs = outputs . step_log_probs sample_id = outputs . predicted_ids else : logits = outputs . rnn_output step_log_probs = tf . no_op ( ) sample_id = outputs . sample_id return logits , sample_id , final_context_state , step_log_probs
Build and run a RNN decoder with a final projection layer.
Build and run a RNN decoder with a final projection layer .
"""Build and run a RNN decoder with a final projection layer.

    Args:
      encoder_outputs: The outputs of encoder for every time step.
      encoder_state: The final state of the encoder.
      hparams: The Hyperparameters configurations.

    Returns:
      A tuple of final logits and final decoder state:
        logits: size [time, batch_size, vocab_size] when time_major=True.
    """ # maximum_iteration: The maximum decoding steps. ## Decoder. # self.init_state_dec = decoder_initial_state #decoder_initial_state = tf.transpose(decoder_initial_state) ## Train or eval # decoder_emp_inp: [max_time, batch_size, num_units] # reshape input from [time, batch_size, 160, full_input_window] to [time * batch_size, 160, full_input_window] # + aa_conf.vocab_size # apply cnn to candidate input, this layer produce output of size # [time * batch_size, filters=32, full_window_size] # [time * batch_size, filters=32, full_window_size] # [time * batch_size, filters * full_window_size] # reshape back to [time, batch_size, filters=32] # 32 * aa_conf.full_aa_window]) # apply dense layer -> [time, batch_size, 512] # Helper # decoder_emb_inp, iterator.target_sequence_length, # Decoder # Dynamic decoding ## Inference # Helper # Decoder # applied per timestep # Dynamic decoding
A tuple of final logits and final decoder state:
logits: size [time, batch_size, vocab_size] when time_major=True.
A tuple of final logits and final decoder state : logits : size [ time batch_size vocab_size ] when time_major = True .
self
encoder_outputs
The outputs of encoder for every time step.
The outputs of encoder for every time step .
encoder_state
The final state of the encoder.
The final state of the encoder .
hparams
The Hyperparameters configurations.
The Hyperparameters configurations .