src/audio2vec_train.py
def train(fn_list,batch_size, memory_dim, seq_len=50, feat_dim=39):
    """ Training seq2seq for number of steps."""
    with tf.Graph().as_default():
        # global_step = tf.Variable(0, trainable=False)
        # get examples and labels for seq2seq #
        examples, labels = batch_pipeline(fn_list, batch_size, feat_dim, seq_len)

        # build a graph that computes the results
        dec_out, dec_memory = inference(examples, batch_size, memory_dim, seq_len,\
            feat_dim)
        # calculate loss
        total_loss = loss(dec_out, examples, seq_len, batch_size, feat_dim)

        ### learning rate decay ###
        learning_rate = tf.placeholder(tf.float32, shape=[])
        tf.summary.scalar("learning rate", learning_rate)

        # build a graph that grains the model with one batch of examples and
        # updates the model parameters
        train_op = train_opt(total_loss, learning_rate, 0.9)
        
        # Create a saver.
        saver = tf.train.Saver(tf.all_variables())
        tf.summary.scalar("RMSE loss", total_loss)
        # Build the summary operation based on the TF collection of Summaries.
        summary_op = tf.summary.merge_all()

        # Build and initialization operation to run below
        init = tf.global_variables_initializer()
        
        # Start running operations on the Graph.
        config = tf.ConfigProto(log_device_placement=False)
        config.gpu_options.allow_growth = True
        sess = tf.Session(config=config)
        sess.run(init)
        sess.graph.finalize()
        # Start the queue runners.
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        
        summary_writer = tf.summary.FileWriter(log_file,sess.graph)

        ### restore the model ###
        ckpt = tf.train.get_checkpoint_state(model_file)
        global_step = 0
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            global_step = \
              int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
        else:
            print ('No checkpoint file found.')
        print ("Model restored.")
        print ("Start batch training.")
        #feed_lr = INITIAL_LEARNING_RATE
        feed_lr = INITIAL_LEARNING_RATE*pow(LEARNING_RATE_DECAY_FACTOR,int(floor(global_step/NUM_EPOCHS_PER_DECAY)))
        ### start training ###
        for step in range(global_step, MAX_STEP+1):
            try:
                
                start_time = time.time()
                _, loss_value= sess.run([train_op,
                    total_loss],feed_dict={learning_rate: feed_lr})
                
                duration = time.time() - start_time
                example_per_sec = batch_size / duration
                epoch = floor(batch_size * step / NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)

                format_str = ('%s: epoch %d, step %d, LR %.5f, loss = %.2f ( %.1f examples/sec;'
                    ' %.3f sec/batch)')
                
                print (format_str % (datetime.now(), epoch, step, feed_lr, loss_value,
                    example_per_sec, float(duration)), end='\r')
                
                # create time line #
                #num_examples_per_step = batch_size
                #tl = timeline.Timeline(run_metadata.step_stats)
                #ctf = tl.generate_chrome_trace_format(show_memory=True)
                if step % 2000 == 0:
                    ckpt = model_file + '/model.ckpt'
                    summary_str = sess.run(summary_op,feed_dict={learning_rate:
                        feed_lr})
                    saver.save(sess, ckpt, global_step=step)
                    summary_writer.add_summary(summary_str,step)
                    summary_writer.flush()
                    #with open('timeline_'+str(step)+'.json','w') as f:
                    #    f.write(ctf)
                if step % NUM_EPOCHS_PER_DECAY == NUM_EPOCHS_PER_DECAY -1 :
                    feed_lr *= LEARNING_RATE_DECAY_FACTOR
            except tf.errors.OutOfRangeError:
                break
        coord.request_stop()
        coord.join(threads)
        summary_writer.flush()
    return
 Training seq2seq for number of steps.
Training seq2seq for number of steps.
Training seq2seq for number of steps .
def train(fn_list,batch_size, memory_dim, seq_len=50, feat_dim=39):
    with tf.Graph().as_default():
        examples, labels = batch_pipeline(fn_list, batch_size, feat_dim, seq_len)
        dec_out, dec_memory = inference(examples, batch_size, memory_dim, seq_len,\
            feat_dim)
        total_loss = loss(dec_out, examples, seq_len, batch_size, feat_dim)
        learning_rate = tf.placeholder(tf.float32, shape=[])
        tf.summary.scalar("learning rate", learning_rate)
        train_op = train_opt(total_loss, learning_rate, 0.9)
        saver = tf.train.Saver(tf.all_variables())
        tf.summary.scalar("RMSE loss", total_loss)
        summary_op = tf.summary.merge_all()
        init = tf.global_variables_initializer()
        config = tf.ConfigProto(log_device_placement=False)
        config.gpu_options.allow_growth = True
        sess = tf.Session(config=config)
        sess.run(init)
        sess.graph.finalize()
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        summary_writer = tf.summary.FileWriter(log_file,sess.graph)
        ckpt = tf.train.get_checkpoint_state(model_file)
        global_step = 0
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            global_step = \
              int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
        else:
            print ('No checkpoint file found.')
        print ("Model restored.")
        print ("Start batch training.")
        feed_lr = INITIAL_LEARNING_RATE*pow(LEARNING_RATE_DECAY_FACTOR,int(floor(global_step/NUM_EPOCHS_PER_DECAY)))
        for step in range(global_step, MAX_STEP+1):
            try:
                start_time = time.time()
                _, loss_value= sess.run([train_op,
                    total_loss],feed_dict={learning_rate: feed_lr})
                duration = time.time() - start_time
                example_per_sec = batch_size / duration
                epoch = floor(batch_size * step / NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
                format_str = ('%s: epoch %d, step %d, LR %.5f, loss = %.2f ( %.1f examples/sec;'
                    ' %.3f sec/batch)')
                print (format_str % (datetime.now(), epoch, step, feed_lr, loss_value,
                    example_per_sec, float(duration)), end='\r')
                if step % 2000 == 0:
                    ckpt = model_file + '/model.ckpt'
                    summary_str = sess.run(summary_op,feed_dict={learning_rate:
                        feed_lr})
                    saver.save(sess, ckpt, global_step=step)
                    summary_writer.add_summary(summary_str,step)
                    summary_writer.flush()
                if step % NUM_EPOCHS_PER_DECAY == NUM_EPOCHS_PER_DECAY -1 :
                    feed_lr *= LEARNING_RATE_DECAY_FACTOR
            except tf.errors.OutOfRangeError:
                break
        coord.request_stop()
        coord.join(threads)
        summary_writer.flush()
    return
def train ( fn_list , batch_size , memory_dim , seq_len = 50 , feat_dim = 39 ) : with tf . Graph ( ) . as_default ( ) : examples , labels = batch_pipeline ( fn_list , batch_size , feat_dim , seq_len ) dec_out , dec_memory = inference ( examples , batch_size , memory_dim , seq_len , feat_dim ) total_loss = loss ( dec_out , examples , seq_len , batch_size , feat_dim ) learning_rate = tf . placeholder ( tf . float32 , shape = [ ] ) tf . summary . scalar ( "learning rate" , learning_rate ) train_op = train_opt ( total_loss , learning_rate , 0.9 ) saver = tf . train . Saver ( tf . all_variables ( ) ) tf . summary . scalar ( "RMSE loss" , total_loss ) summary_op = tf . summary . merge_all ( ) init = tf . global_variables_initializer ( ) config = tf . ConfigProto ( log_device_placement = False ) config . gpu_options . allow_growth = True sess = tf . Session ( config = config ) sess . run ( init ) sess . graph . finalize ( ) coord = tf . train . Coordinator ( ) threads = tf . train . start_queue_runners ( sess = sess , coord = coord ) summary_writer = tf . summary . FileWriter ( log_file , sess . graph ) ckpt = tf . train . get_checkpoint_state ( model_file ) global_step = 0 if ckpt and ckpt . model_checkpoint_path : saver . restore ( sess , ckpt . model_checkpoint_path ) global_step = int ( ckpt . model_checkpoint_path . split ( '/' ) [ - 1 ] . split ( '-' ) [ - 1 ] ) else : print ( 'No checkpoint file found.' ) print ( "Model restored." ) print ( "Start batch training." ) feed_lr = INITIAL_LEARNING_RATE * pow ( LEARNING_RATE_DECAY_FACTOR , int ( floor ( global_step / NUM_EPOCHS_PER_DECAY ) ) ) for step in range ( global_step , MAX_STEP + 1 ) : try : start_time = time . time ( ) _ , loss_value = sess . run ( [ train_op , total_loss ] , feed_dict = { learning_rate : feed_lr } ) duration = time . time ( ) - start_time example_per_sec = batch_size / duration epoch = floor ( batch_size * step / NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN ) format_str = ( '%s: epoch %d, step %d, LR %.5f, loss = %.2f ( %.1f examples/sec;' ' %.3f sec/batch)' ) print ( format_str % ( datetime . now ( ) , epoch , step , feed_lr , loss_value , example_per_sec , float ( duration ) ) , end = '\r' ) if step % 2000 == 0 : ckpt = model_file + '/model.ckpt' summary_str = sess . run ( summary_op , feed_dict = { learning_rate : feed_lr } ) saver . save ( sess , ckpt , global_step = step ) summary_writer . add_summary ( summary_str , step ) summary_writer . flush ( ) if step % NUM_EPOCHS_PER_DECAY == NUM_EPOCHS_PER_DECAY - 1 : feed_lr *= LEARNING_RATE_DECAY_FACTOR except tf . errors . OutOfRangeError : break coord . request_stop ( ) coord . join ( threads ) summary_writer . flush ( ) return
Training seq2seq for number of steps.
Training seq2seq for number of steps .
""" Training seq2seq for number of steps.""" # global_step = tf.Variable(0, trainable=False) # get examples and labels for seq2seq # # build a graph that computes the results # calculate loss ### learning rate decay ### # build a graph that grains the model with one batch of examples and # updates the model parameters # Create a saver. # Build the summary operation based on the TF collection of Summaries. # Build and initialization operation to run below # Start running operations on the Graph. # Start the queue runners. ### restore the model ### #feed_lr = INITIAL_LEARNING_RATE ### start training ### # create time line # #num_examples_per_step = batch_size #tl = timeline.Timeline(run_metadata.step_stats) #ctf = tl.generate_chrome_trace_format(show_memory=True) #with open('timeline_'+str(step)+'.json','w') as f: #    f.write(ctf)
fn_list
batch_size
memory_dim
seq_len
feat_dim