courses/machine_learning/deepdive/09_sequence/word2vec/word2vec.py
def forward(self, examples, labels):
    """Build the graph for the forward pass."""
    opts = self._options

    # Declare all variables we need.
    # Embedding: [vocab_size, emb_dim]
    init_width = 0.5 / opts.emb_dim
    emb = tf.Variable(
        tf.random_uniform(
            [opts.vocab_size, opts.emb_dim], -init_width, init_width),
        name="emb")
    self._emb = emb

    # Softmax weight: [vocab_size, emb_dim]. Transposed.
    sm_w_t = tf.Variable(
        tf.zeros([opts.vocab_size, opts.emb_dim]),
        name="sm_w_t")

    # Softmax bias: [vocab_size].
    sm_b = tf.Variable(tf.zeros([opts.vocab_size]), name="sm_b")

    # Global step: scalar, i.e., shape [].
    self.global_step = tf.Variable(0, name="global_step")

    # Nodes to compute the nce loss w/ candidate sampling.
    labels_matrix = tf.reshape(
        tf.cast(labels,
                dtype=tf.int64),
        [opts.batch_size, 1])

    # Negative sampling.
    sampled_ids, _, _ = (tf.nn.fixed_unigram_candidate_sampler(
        true_classes=labels_matrix,
        num_true=1,
        num_sampled=opts.num_samples,
        unique=True,
        range_max=opts.vocab_size,
        distortion=0.75,
        unigrams=opts.vocab_counts.tolist()))

    # Embeddings for examples: [batch_size, emb_dim]
    example_emb = tf.nn.embedding_lookup(emb, examples)

    # Weights for labels: [batch_size, emb_dim]
    true_w = tf.nn.embedding_lookup(sm_w_t, labels)
    # Biases for labels: [batch_size, 1]
    true_b = tf.nn.embedding_lookup(sm_b, labels)

    # Weights for sampled ids: [num_sampled, emb_dim]
    sampled_w = tf.nn.embedding_lookup(sm_w_t, sampled_ids)
    # Biases for sampled ids: [num_sampled, 1]
    sampled_b = tf.nn.embedding_lookup(sm_b, sampled_ids)

    # True logits: [batch_size, 1]
    true_logits = tf.reduce_sum(tf.multiply(example_emb, true_w), 1) + true_b

    # Sampled logits: [batch_size, num_sampled]
    # We replicate sampled noise labels for all examples in the batch
    # using the matmul.
    sampled_b_vec = tf.reshape(sampled_b, [opts.num_samples])
    sampled_logits = tf.matmul(example_emb,
                               sampled_w,
                               transpose_b=True) + sampled_b_vec
    return true_logits, sampled_logits
Build the graph for the forward pass.
Build the graph for the forward pass.
Build the graph for the forward pass .
def forward(self, examples, labels):
    opts = self._options
    init_width = 0.5 / opts.emb_dim
    emb = tf.Variable(
        tf.random_uniform(
            [opts.vocab_size, opts.emb_dim], -init_width, init_width),
        name="emb")
    self._emb = emb
    sm_w_t = tf.Variable(
        tf.zeros([opts.vocab_size, opts.emb_dim]),
        name="sm_w_t")
    sm_b = tf.Variable(tf.zeros([opts.vocab_size]), name="sm_b")
    self.global_step = tf.Variable(0, name="global_step")
    labels_matrix = tf.reshape(
        tf.cast(labels,
                dtype=tf.int64),
        [opts.batch_size, 1])
    sampled_ids, _, _ = (tf.nn.fixed_unigram_candidate_sampler(
        true_classes=labels_matrix,
        num_true=1,
        num_sampled=opts.num_samples,
        unique=True,
        range_max=opts.vocab_size,
        distortion=0.75,
        unigrams=opts.vocab_counts.tolist()))
    example_emb = tf.nn.embedding_lookup(emb, examples)
    true_w = tf.nn.embedding_lookup(sm_w_t, labels)
    true_b = tf.nn.embedding_lookup(sm_b, labels)
    sampled_w = tf.nn.embedding_lookup(sm_w_t, sampled_ids)
    sampled_b = tf.nn.embedding_lookup(sm_b, sampled_ids)
    true_logits = tf.reduce_sum(tf.multiply(example_emb, true_w), 1) + true_b
    sampled_b_vec = tf.reshape(sampled_b, [opts.num_samples])
    sampled_logits = tf.matmul(example_emb,
                               sampled_w,
                               transpose_b=True) + sampled_b_vec
    return true_logits, sampled_logits
def forward ( self , examples , labels ) : opts = self . _options init_width = 0.5 / opts . emb_dim emb = tf . Variable ( tf . random_uniform ( [ opts . vocab_size , opts . emb_dim ] , - init_width , init_width ) , name = "emb" ) self . _emb = emb sm_w_t = tf . Variable ( tf . zeros ( [ opts . vocab_size , opts . emb_dim ] ) , name = "sm_w_t" ) sm_b = tf . Variable ( tf . zeros ( [ opts . vocab_size ] ) , name = "sm_b" ) self . global_step = tf . Variable ( 0 , name = "global_step" ) labels_matrix = tf . reshape ( tf . cast ( labels , dtype = tf . int64 ) , [ opts . batch_size , 1 ] ) sampled_ids , _ , _ = ( tf . nn . fixed_unigram_candidate_sampler ( true_classes = labels_matrix , num_true = 1 , num_sampled = opts . num_samples , unique = True , range_max = opts . vocab_size , distortion = 0.75 , unigrams = opts . vocab_counts . tolist ( ) ) ) example_emb = tf . nn . embedding_lookup ( emb , examples ) true_w = tf . nn . embedding_lookup ( sm_w_t , labels ) true_b = tf . nn . embedding_lookup ( sm_b , labels ) sampled_w = tf . nn . embedding_lookup ( sm_w_t , sampled_ids ) sampled_b = tf . nn . embedding_lookup ( sm_b , sampled_ids ) true_logits = tf . reduce_sum ( tf . multiply ( example_emb , true_w ) , 1 ) + true_b sampled_b_vec = tf . reshape ( sampled_b , [ opts . num_samples ] ) sampled_logits = tf . matmul ( example_emb , sampled_w , transpose_b = True ) + sampled_b_vec return true_logits , sampled_logits
Build the graph for the forward pass.
Build the graph for the forward pass .
"""Build the graph for the forward pass.""" # Declare all variables we need. # Embedding: [vocab_size, emb_dim] # Softmax weight: [vocab_size, emb_dim]. Transposed. # Softmax bias: [vocab_size]. # Global step: scalar, i.e., shape []. # Nodes to compute the nce loss w/ candidate sampling. # Negative sampling. # Embeddings for examples: [batch_size, emb_dim] # Weights for labels: [batch_size, emb_dim] # Biases for labels: [batch_size, 1] # Weights for sampled ids: [num_sampled, emb_dim] # Biases for sampled ids: [num_sampled, 1] # True logits: [batch_size, 1] # Sampled logits: [batch_size, num_sampled] # We replicate sampled noise labels for all examples in the batch # using the matmul.
self
examples
labels