# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### refer code 
# https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py
#
# https://hunkim.github.io/ml/
#

# ### import packages

# +
import os
os.environ["CUDA_VISIBLE_DEVICES"]="4" #4번 DEVICES만 사용하여 다른 메모리를 낭비를 방지
import time

import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split

#costomized seq2seq cell
import copy
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import variable_scope

# +
#=======================================Hyperparameter 
learning_rate = 0.001
total_epoch = 1000
batch_size = 100 
dropout = 0.5
hidden_size = 200
index = 0

feed_train = {}
feed_val = {}
outputs = []
targets = []
DATAS = []
data_size = 50

#훈련에 사용할 데이터 이름을 배열로 저장한다. 
traindatasets = ['Plane', 'Gun_Point', 'ArrowHead', 'WordsSynonyms', 'ToeSegmentation1', 'FISH', 'ShapeletSim', 'ShapesAll', 'SonyAIBORobotSurfaceII',
             'Lighting7', 'ToeSegmentation2', 'DiatomSizeReduction', 'Ham', 'SonyAIBORobotSurface', 'TwoLeadECG', 'FacesUCR']


# +
def customized_rnn_seq2seq(encoder_inputs,
                          decoder_inputs,
                          cell,
                          dtype=dtypes.float32,
                          scope=None):
    
    with variable_scope.variable_scope(scope or "basic_rnn_seq2seq"):
        enc_cell = copy.deepcopy(cell)
        encoder_outputs, enc_state = tf.contrib.rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype)
    return customized_rnn_decoder(encoder_outputs, decoder_inputs, enc_state, cell)

def customized_rnn_decoder(encoder_outputs,
                decoder_inputs,
                initial_state,
                cell,
                loop_function=None,
                scope=None):
    
    with variable_scope.variable_scope(scope or "rnn_decoder"):
        state = initial_state
        outputs = []
        prev = None
        for i, inp in enumerate(decoder_inputs):
            if loop_function is not None and prev is not None:
                with variable_scope.variable_scope("loop_function", reuse=True):
                    inp = loop_function(prev, i)
            if i > 0:
                variable_scope.get_variable_scope().reuse_variables()
            output, state = cell(inp, state)
            outputs.append(output)
            if loop_function is not None:
                prev = output
    return encoder_outputs, outputs, state

def data_maker(dataset):

    #train 숫자가 너무 작아서 concatenate이후 train_test_split를 이용해 다시 나눈다.
    datadir = '/home/sohee/UCR_TS_Archive_2015' + '/' + dataset + '/' + dataset
    data_train = np.loadtxt(datadir+'_TRAIN', delimiter=',')
    data_test = np.loadtxt(datadir+'_TEST', delimiter=',')
    DATA = np.concatenate((data_train,data_test),axis=0)
    
    #특정 데이터의 y lable이 1부터 시작해 range error 발생.첫 인덱스가 1이면 1을 빼서 0으로 만듬
    X_data = DATA[:,1:]
    y_data = DATA[:,[0]]
    y_first_index = int(np.unique(y_data)[0])
    if int(y_first_index) == 1:
        y_data = DATA[:,[0]]-1
    DATA = np.concatenate((y_data, X_data), axis=1)
    
    #50행마다 잘라서 DATAS 배열에 저정
    iter = DATA.shape[0]//data_size
    for i in range(iter):
        CUT_DATA = DATA[50*i:50*(i+1),:]
        DATAS.append(CUT_DATA)
    
def seq2seq_maker(index, DATA):

    X_data = DATA[:,1:]
    y_data = DATA[:,0]
    
    X_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.2, random_state=1)
    
    encoder_input = tf.placeholder(tf.float32, [None, X_train.shape[1]]) 
    decoder_input = tf.placeholder(tf.float32, [None, X_train.shape[1]])
    target = tf.placeholder(tf.int64, [None])

    targets.append(target)
    
    with tf.variable_scope("rnn_"+str(index)):
        cell = tf.contrib.rnn.GRUCell(num_units=hidden_size)
        cell = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=0.5)
        #output, states = tf.contrib.legacy_seq2seq.basic_rnn_seq2seq([encoder_input], [decoder_input], cell)
        en_outputs, de_outputs, state = customized_rnn_seq2seq([encoder_input], [decoder_input], cell)
        #print("encoder_outputs", encoder_outputs)
        #print("decoder_outputs", decoder_outputs)
        de_outputs = tf.reshape(de_outputs, [-1, hidden_size]) #3D -> 2D
        outputs.append(de_outputs)
    
    feed_train[encoder_input] = X_train
    feed_train[decoder_input] = X_train
    feed_train[target] = y_train
    
    feed_val[encoder_input] = X_val
    feed_val[decoder_input] = X_val
    feed_val[target] = y_val


# +
import copy
#data_maker('Plane')        
for dataset in traindatasets:
    data_maker(dataset)    
    
for index, DATA in enumerate(DATAS):
    seq2seq_maker(index, DATA)

# +
W = tf.Variable(tf.random_normal([hidden_size, 100]), name="W") 
b = tf.Variable(tf.random_normal([100]), name="b") #최대 클래수의 갯수인 60보다는 커야 range 에러 안생김
print("W", W)
print("b", b)
logits = [tf.matmul(output, W) + b for output in outputs]

with tf.variable_scope("cost"):
    loss = []
    for logit, target in zip(logits, targets):
        loss.append(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logit, labels=target))
    #cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets))
    cost= tf.reduce_mean(loss)
    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) 
    
#with tf.variable_scope("eval"):
#    prediction = tf.argmax(tf.nn.softmax(logits), 1) 
#    correct_prediction = tf.equal(prediction, targets)#one-hot을 안쓰면 target에는 argmax할 필요없음
#    accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    

# +
saver = tf.train.Saver()

with tf.Session() as sess:
    tf.global_variables_initializer().run()
    
    print(sess.run("W:0"))
    print(sess.run("b:0"))
    print("============================sess init")
    start_time = time.time()
    train_losses = []
    valid_losses = []
    for epoch in range(total_epoch):
        _, tr_loss = sess.run([optimizer, cost], feed_dict = feed_train)
        _, va_loss = sess.run([optimizer, cost], feed_dict = feed_val)
        train_losses += [tr_loss]
        valid_losses += [va_loss]
        if epoch % 100 == 0:  
            print("Epoch {}/{} took {:.3f}s".format(epoch + 1, total_epoch,time.time() - start_time))
            print("  Train      loss : %.6f"%(train_losses[epoch]))
            print("  Validation loss : %.6f"%(valid_losses[epoch]))
    print("It took", time.time() - start_time, "seconds to train for", total_epoch, "epochs.")        
    print("============================ training end")
    
    #loss  그래프를 확인한다. 
    plt.plot(train_losses, '-b', label='Train loss')
    plt.plot(valid_losses, '-r', label='Valid loss')
    plt.legend(loc=0)
    plt.title('Loss graph')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.show()
    #print('Accuracy:', sess.run(accuracy_op, feed_dict=feed_dict))
    
    #pred = sess.run(prediction, feed_dict=feed_dict)
    #for p, i in zip(pred, y_test.flatten()):
    #    print("[{}] Prediction: {} True Y: {}".format(p == int(i), p, int(i)))
        
    print(sess.run("W:0"))
    print(sess.run("b:0"))
    save_path = saver.save(sess, "./180123.ckpt")
    print("model saved!")
# -

tf.reset_default_graph()
saver = tf.train.import_meta_graph("./180123.ckpt.meta")

# +
learning_rate = 0.001
total_epoch = 1000
batch_size = 100 
dropout = 0.5
hidden_size = 200
index = 0

dataset = 'ChlorineConcentration'
datadir = '/home/sohee/UCR_TS_Archive_2015' + '/' + dataset + '/' + dataset
data_train = np.loadtxt(datadir+'_TRAIN', delimiter=',')
data_test = np.loadtxt(datadir+'_TEST', delimiter=',')

#train 숫자가 너무 작아서 concatenate이후 train_test_split를 이용해 다시 나눈다.
DATA = np.concatenate((data_train,data_test),axis=0)
X_data_n = DATA[:,1:]
y_data_n = DATA[:,0]

#특정 데이터의 y lable이 1부터 시작해 range error 발생.첫 인덱스가 1이면 1을 빼서 0으로 만듬
y_first_index = int(np.unique(y_data_n)[0])
if int(y_first_index) == 1:
    y_data_n = DATA[:,0]-1

X_train_n, X_test_n, y_train_n, y_test_n = train_test_split(X_data_n, y_data_n, test_size=0.2, random_state=1)
X_train_n, X_val_n, y_train_n, y_val_n = train_test_split(X_train_n, y_train_n, test_size=0.2, random_state=1)

n_variable = X_train_n.shape[1]

# placeholder
encoder_inputs = tf.placeholder(tf.float32, [None, n_variable], name="encoder_inputs") 
decoder_inputs = tf.placeholder(tf.float32, [None, n_variable], name="decoder_inputs")
targets = tf.placeholder(tf.int64, [None], name="targets")

with tf.variable_scope("seq2seq"+dataset):  
    cell = tf.contrib.rnn.GRUCell(num_units=hidden_size)
    outputs, states = tf.contrib.legacy_seq2seq.basic_rnn_seq2seq([encoder_inputs], [decoder_inputs], cell)
    outputs = tf.reshape(outputs, [-1, hidden_size]) #3D -> 2D
    print("outputs", outputs)  # output 모양이 항상 [? , hidden_size]으로 고정됨

    W = tf.get_default_graph().get_tensor_by_name("W:0")
    b = tf.get_default_graph().get_tensor_by_name("b:0")
    print("W", W)
    print("b", b)
    logits = tf.matmul(outputs, W) + b

with tf.variable_scope("cost"+dataset):
    cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets))
    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) 

with tf.variable_scope("eval"+dataset):
    prediction = tf.argmax(tf.nn.softmax(logits), 1) 
    correct_prediction = tf.equal(prediction, targets)#one-hot을 안쓰면 target에는 argmax할 필요없음
    accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# +
train_losses = []
val_losses = []

new_saver = tf.train.Saver()

with tf.Session() as sess:
    tf.global_variables_initializer().run()
    saver.restore(sess, "./180123.ckpt")
    print("... check point loaded")
    print(sess.run("W:0"))
    print(sess.run("b:0"))
    print("=============================== training", dataset)
    start_time = time.time()
    # 비용함수을 학습시킨다. 
    for epoch in range(total_epoch):
        #Minibatch 
        batch_mask = np.random.choice(len(X_train_n), batch_size)
        X_batch = X_train_n[batch_mask]
        y_batch = y_train_n[batch_mask]

        # Compute the losses, 훈련을 통해 w,b가 계속 조정되고 그로 인해 변하는 정확성이나 비용함수를 측정하는 것
        _, tr_loss = sess.run([optimizer, cost], feed_dict={encoder_inputs: X_batch, decoder_inputs: X_batch, targets: y_batch})
        _, val_loss = sess.run([optimizer, cost], feed_dict={encoder_inputs: X_val_n, decoder_inputs: X_val_n,targets: y_val_n})
        # Log the losses
        train_losses += [tr_loss]
        val_losses += [val_loss]
  
        if epoch % 100 == 0:  
            print("Epoch {}/1000 took {:.3f}s".format(epoch + 1, time.time() - start_time))
            print("  Train loss : %.6f"%(train_losses[epoch]))
            print("  valid loss : %.6f"%(val_losses[epoch]))

    print("============================ training end")
    print('Accuracy:', sess.run(accuracy_op, feed_dict={encoder_inputs: X_test_n, decoder_inputs: X_test_n, targets: y_test_n}))
    
    pred = sess.run(prediction, feed_dict={encoder_inputs: X_test_n, decoder_inputs: X_test_n})
    for p, i in zip(pred, y_test_n.flatten()):
        print("[{}] Prediction: {} True Y: {}".format(p == int(i), p, int(i)))

StarcoderdataJupyterScriptsDedupFiltered