#方案2
orders = tle.get_orders()
users_orders = tle.get_users_orders('prior')
# 将product_id转换为str
users_products_matrix = users_orders.groupby(['user_id'])['product_id'].apply(utils.series_to_str)
# 构造vocabulary
tf = CountVectorizer(analyzer = 'word', lowercase = False, max_df=0.95, min_df=2,)
tf_matrix = tf.fit_transform(users_products_matrix.values)
tf_feature_names = tf.get_feature_names()

#with open(DATA_DIR + 'user_tf_matrix', 'wb') as f:
#    pickle.dump(tf_matrix, f, pickle.HIGHEST_PROTOCOL)

products = tle.get_items('products')
aisles = tle.get_items('aisles')
departments = tle.get_items('departments')

product_a = pd.merge(products, aisles, on = ['aisle_id'], how = 'left')
product_ad = pd.merge(product_a, departments, on = ['department_id'], how = 'left')

del product_ad['aisle_id']
del product_ad['department_id']

product_ad['chain_product_name'] = product_ad['department'] + ' _ ' +\
                                   product_ad['aisle'] + ' _ ' +\
                                   product_ad['product_name'] 

tf_product_names = [] #商品名列表
for pid in tf_feature_names:
    tf_product_names.append(product_ad[product_ad.product_id == int(pid)]['chain_product_name'].values[0])

tf_info = pd.DataFrame({'pid':tf_feature_names, 'pname':tf_product_names})

#with open(DATA_DIR + 'user_tf_matrix_info.pkl', 'wb') as f:
#    pickle.dump(tf_info, f, pickle.HIGHEST_PROTOCOL)

"""
Explanation: LDA预处理
方案 1

将每张订单作为一个document

方案2

将每个用户作为一个document
将每个product作为一个Document
将每个aisle作为一个Document
将每个department作为Document

方案3

product name
无结构文本

<font color=red>方案2 part 1</font>
End of explanation
"""


orders = tle.get_orders()
users_orders = tle.get_users_orders('prior')

def pad_tf(pad, users_orders):

    # 将product_id转换为str
    pad_users_matrix = users_orders.groupby([pad])['user_id'].apply(utils.series_to_str)
    # 构造vocabulary
    tf = CountVectorizer(analyzer = 'word', lowercase = False, max_df=0.95, min_df=2,)
    tf_matrix = tf.fit_transform(pad_users_matrix.values)
    tf_feature_names = tf.get_feature_names()
    tf_info = pd.DataFrame({'term_id':np.arange(len(tf_feature_names)), 'user_id':tf_feature_names})
    
    with open(constants.FEAT_DATA_DIR + pad[:-3] + '_tf_matrix', 'wb') as f:
        pickle.dump(tf_matrix, f, pickle.HIGHEST_PROTOCOL)
    with open(constants.FEAT_DATA_DIR + pad[:-3] + '_tf_info', 'wb') as f:
        pickle.dump(tf_info, f, pickle.HIGHEST_PROTOCOL)

for pad in ['product_id', 'aisle_id', 'department_id']:
    pad_tf(pad, users_orders)

pad = 'aisle_id'

with open(constants.FEAT_DATA_DIR + pad[:-3] + '_tf_matrix', 'rb') as f:
    a_tf_matrix = pickle.load(f)    

a_tf_matrix.shape

pad = 'product_id'

with open(constants.FEAT_DATA_DIR + pad[:-3] + '_tf_matrix', 'rb') as f:
    p_tf_matrix = pickle.load(f)    

p_tf_matrix.shape

"""
Explanation: <font color=red>方案2 part 2 </font>
End of explanation
"""


from gensim.models.ldamulticore import LdaMulticore
from gensim import corpora

import constants

# raw data
tle = transactions.TransLogExtractor(constants.RAW_DATA_DIR, constants.FEAT_DATA_DIR)
users_orders = tle.get_users_orders('prior')

%%time

def pad_tf(pad, users_orders):
    pad_user = users_orders.groupby([pad])['user_id'].apply(utils.series_to_str) # convert into str
    pad_user = [doc.split() for doc in pad_user.values] # split into arrays
    dictionary = corpora.Dictionary(pad_user) # create dictionary
    pad_term_matrix = [dictionary.doc2bow(doc) for doc in pad_user]
    return dictionary, pad_term_matrix

for pad in ['product_id', 'aisle_id', 'department_id']:
    p_dict, p_term_matrix = pad_tf(pad, users_orders)
    with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_dict', 'wb') as f:
        pickle.dump(p_dict, f, pickle.HIGHEST_PROTOCOL)
    with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'wb') as f:
        pickle.dump(p_term_matrix, f, pickle.HIGHEST_PROTOCOL)

pad = 'department_id'
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'rb') as f:
    p_term_matrix = pickle.load(f)

%%time
pad = 'product_id'
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'rb') as f:
    p_term_matrix = pickle.load(f)
for n in [10, 60, 110, 160, 210]:
    print('number of topics: %d'%n)
    lda = LdaMulticore.load(constants.LDA_DIR + 'p_gensim_lda_%d'%n)
    print(lda.log_perplexity(p_term_matrix, total_docs=1677))


num_topic = 10
lda = LdaMulticore.load(constants.LDA_DIR + 'p_gensim_lda_%d'%num_topic)

pad = 'product_id'
pad_user = users_orders.groupby([pad])['user_id'].apply(utils.series_to_str)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_dict', 'rb') as f:
    p_dict = pickle.load(f)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'rb') as f:
    p_term_matrix = pickle.load(f)

p_topics = [[v for k,v in lda.get_document_topics(p, minimum_phi_value=0, minimum_probability=0)] for p in p_term_matrix]
p_topics = pd.DataFrame(p_topics, columns = ['p_topic_%d'%n for n in range(num_topic)])
p_topics['product_id'] = pad_user.index.values

user_id = [int(token) for word_id, token in p_dict.iteritems()]
u_topics = [{k:v for k,v in lda.get_term_topics(word_id, minimum_probability=0)} for word_id, _ in p_dict.iteritems()]
u_topics = pd.DataFrame(u_topics).fillna(0)
u_topics.columns = ['u_topic_%d'%i for i in range(num_topic)]
u_topics = u_topics / u_topics.sum() # column normalization, make sure each topic sums to 1
u_topics = (u_topics.transpose() / u_topics.transpose().sum()).transpose() # row normalization
u_topics['user_id'] = user_id

with open(constants.FEAT_DATA_DIR + 'pad_p_topic.pkl', 'wb') as f:
    pickle.dump(p_topics, f, pickle.HIGHEST_PROTOCOL)
with open(constants.FEAT_DATA_DIR + 'pad_p_u_topic.pkl', 'wb') as f:
    pickle.dump(u_topics, f, pickle.HIGHEST_PROTOCOL) 

num_topic = 10
lda = LdaMulticore.load(constants.LDA_DIR + 'a_gensim_lda_%d'%num_topic)

pad = 'aisle_id'
pad_user = users_orders.groupby([pad])['user_id'].apply(utils.series_to_str)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_dict', 'rb') as f:
    p_dict = pickle.load(f)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'rb') as f:
    p_term_matrix = pickle.load(f)

p_topics = [[v for k,v in lda.get_document_topics(p, minimum_phi_value=0, minimum_probability=0)] for p in p_term_matrix]
p_topics = pd.DataFrame(p_topics, columns = ['p_topic_%d'%n for n in range(num_topic)])
p_topics['aisle_id'] = pad_user.index.values

user_id = [int(token) for word_id, token in p_dict.iteritems()]
u_topics = [{k:v for k,v in lda.get_term_topics(word_id, minimum_probability=0)} for word_id, _ in p_dict.iteritems()]
u_topics = pd.DataFrame(u_topics).fillna(0)
u_topics.columns = ['u_topic_%d'%i for i in range(num_topic)]
u_topics = u_topics / u_topics.sum() # column normalization, make sure each topic sums to 1
u_topics = (u_topics.transpose() / u_topics.transpose().sum()).transpose() # row normalization
u_topics['user_id'] = user_id

with open(constants.FEAT_DATA_DIR + 'pad_a_topic.pkl', 'wb') as f:
    pickle.dump(p_topics, f, pickle.HIGHEST_PROTOCOL)
with open(constants.FEAT_DATA_DIR + 'pad_a_u_topic.pkl', 'wb') as f:
    pickle.dump(u_topics, f, pickle.HIGHEST_PROTOCOL) 

num_topic = 4 
lda = LdaMulticore.load(constants.LDA_DIR + 'd_gensim_lda_%d'%num_topic)

pad = 'department_id'
pad_user = users_orders.groupby([pad])['user_id'].apply(utils.series_to_str)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_dict', 'rb') as f:
    p_dict = pickle.load(f)
with open(constants.FEAT_DATA_DIR + pad[:-3] + '_gensim_tf', 'rb') as f:
    p_term_matrix = pickle.load(f)

p_topics = [[v for k,v in lda.get_document_topics(p, minimum_phi_value=0, minimum_probability=0)] for p in p_term_matrix]
p_topics = pd.DataFrame(p_topics, columns = ['p_topic_%d'%n for n in range(num_topic)])
p_topics[pad] = pad_user.index.values

user_id = [int(token) for word_id, token in p_dict.iteritems()]
u_topics = [{k:v for k,v in lda.get_term_topics(word_id, minimum_probability=0)} for word_id, _ in p_dict.iteritems()]
u_topics = pd.DataFrame(u_topics).fillna(0)
# u_topics[4] = 0 # column 4 missing
u_topics.columns = ['u_topic_%d'%i for i in range(num_topic)]
u_topics = u_topics / u_topics.sum() # column normalization, make sure each topic sums to 1
u_topics = (u_topics.transpose() / u_topics.transpose().sum()).transpose() # row normalization
u_topics['user_id'] = user_id

with open(constants.FEAT_DATA_DIR + 'pad_d_topic.pkl', 'wb') as f:
    pickle.dump(p_topics, f, pickle.HIGHEST_PROTOCOL)
with open(constants.FEAT_DATA_DIR + 'pad_d_u_topic.pkl', 'wb') as f:
    pickle.dump(u_topics, f, pickle.HIGHEST_PROTOCOL) 

ud = tle.craft_up_distance(filepath = ['pad_d_topic.pkl', 'pad_d_u_topic.pkl'], num_topic = 8, pad = 'department_id')

ud = tle.craft_up_distance(filepath = ['pad_a_topic.pkl', 'pad_a_u_topic.pkl'], num_topic = 10, pad = 'aisle_id')

ud = tle.craft_up_distance(filepath = ['pad_p_topic.pkl', 'pad_p_u_topic.pkl'], num_topic = 10, pad = 'product_id')

?? tle.craft_up_distance

"""
Explanation: 方案2 <font color=red> Gensim </font>

per word bound
困惑度小，模型好，perplexity $2^{-bound}$
bound大，模型好
End of explanation
"""


#方案3
order_product_names = pd.merge(order_products_all[['order_id', 'product_id']],
                               products[['product_id', 'product_name']],
                               on = ['product_id'],
                               how = 'left')

# 8mins
order_pnames_matrix = order_product_names.groupby(['order_id'])['product_name'].aggregate('sum')

order_pnames_matrix.head(5)

%%time
tf = CountVectorizer(analyzer = 'word', min_df=10, token_pattern='(?u)\\b[a-zA-Z]\\w+\\b')
tf_matrix = tf.fit_transform(order_pnames_matrix.values)
tf_feature_names = tf.get_feature_names()

'crisp' in tf_feature_names

tf_matrix.shape

"""
Explanation: 方案3 需要处理无结构化的文本
End of explanation
"""


with open(DATA_DIR + 'user_tf_matrix', 'rb') as f:
    user_tf_matrix = pickle.load(f)

%%time
n_topics = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
n_top_words = 10
scores = []
for n in n_topics:
    print("number of topics:%d"%n)
    with open(DATA_DIR + 'lda_%d.model'%n, 'rb') as f:
        lda = pickle.load(f)
        scores.append(lda.score(user_tf_matrix))
        print("log likelihood:%f"%scores[-1])
        #print_top_words(lda, tf_product_names, n_top_words)

max(scores)


"""
Explanation: LDA Topic数目选择
number of topics:10
log likelihood:-482209266.283346
number of topics:20
log likelihood:-739555562.008604
number of topics:30
log likelihood:-1005367108.142659
number of topics:40
log likelihood:-1293773643.990032
number of topics:50
log likelihood:-1578475501.104584
number of topics:60
log likelihood:-1891792320.600158
number of topics:70
log likelihood:-2185429399.135661
number of topics:80
log likelihood:-2499275522.415354
number of topics:90
log likelihood:-2814907367.346162
number of topics:100
log likelihood:-3124264684.650005
End of explanation
"""


with open(constants.LDA_DIR + 'lda_22.model', 'rb') as f:
    lda = pickle.load(f)

with open(constants.FEAT_DATA_DIR + 'user_tf_matrix_info.pkl', 'rb') as f:
    tf_info = pickle.load(f)

with open(constants.FEAT_DATA_DIR + 'user_tf_matrix', 'rb') as f:
    user_tf_matrix = pickle.load(f)

def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print("\n".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    print()

def csv_top_words(model, feature_names, n_top_words):
    topic_content = {}
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        content = [feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]
        # print("\n".join(content))
        topic_content['topic_%s'%topic_idx] = content
    res = pd.DataFrame(topic_content)
    res.to_csv('lda_topic_content.csv')
    print()
    return res

lda.components_[0][:10:1]

print_top_words(lda, tf_info.pname, 20)

"""
Explanation: <font color=red>Post-LDA Topic内容</font>
End of explanation
"""


NUM_TOPIC = 22
tf_feature_names = tf_info.pid.values
with open(constants.LDA_DIR + 'lda_%d.model'%NUM_TOPIC, 'rb') as f:# replace 10
    lda = pickle.load(f)

# 每个话题是一个商品上的分布, 除以topic的总比重得到每个单词对每个Topic的贡献比重
# np.newaxis turn (22,) into (22,1)
norm_comp = lda.components_ / lda.components_.sum(axis=1)[:, np.newaxis]

# 每个商品也可对应一个话题的分布，对每个单词的分布归一化使得概率和为1
norm_comp = norm_comp / norm_comp.sum(axis=0)

topic_product = pd.DataFrame(norm_comp.transpose(), columns = ["topic_%d"%x for x in range(NUM_TOPIC)])
topic_product['product_id'] = [int(x) for x in tf_feature_names]

user_topic = lda.transform(user_tf_matrix)
user_topic = pd.DataFrame(user_topic, columns = ["topic_%d"%x for x in range(NUM_TOPIC)])
user_id = users_products_matrix.index.values
user_topic['user_id'] = user_id

with open(DATA_DIR + 'user_topic_%d.pkl'%NUM_TOPIC, 'wb') as f:
    pickle.dump(user_topic, f, pickle.HIGHEST_PROTOCOL)
with open(DATA_DIR + 'topic_product_%d.pkl'%NUM_TOPIC, 'wb') as f:
    pickle.dump(topic_product, f, pickle.HIGHEST_PROTOCOL)

with open(DATA_DIR + 'user_topic_%d.pkl'%NUM_TOPIC, 'rb') as f:
    user_topic = pickle.load(f)
with open(DATA_DIR + 'topic_product_%d.pkl'%NUM_TOPIC, 'rb') as f:
    topic_product = pickle.load(f)

user_topic.head(5)

topic_product.describe()

"""
Explanation: 方案2 每个用户的<font color=red>Embeded Topic Space</font>表达

用户对应一个Topic, 各个分量的scale都比较小, 是一个分布，l1范数和为1
商品对应一个Topic，各个分量的scale都很大，<font color=red>Normalize Components</font>
<font color=blue> option 1 </font>K_u NN 提供Candidate商品
欧氏距离
概率


<font color=blue> option 2 </font>embed vector直接作为特征
Time Weighted?
End of explanation
"""


with open(DATA_DIR + 'user_product.pkl', 'rb') as f:
    user_product = pickle.load(f)

with open(DATA_DIR + 'user_reorder_est.pkl', 'rb') as f:
    avg_reorder_est = pickle.load(f)

train_orders = orders[orders.eval_set == 'train']

#%%time
u_nnp = []
cnt = 0
for u in train_orders.user_id:
    cnt += 1
    if cnt % 10000 == 0:
        print("Nearest Product Search for %dth user"%cnt)
    # extract user u's topic
    u_topic = user_topic[user_topic.user_id == u][["topic_%d"%x for x in range(10)]]
    # extract user u's product list
    u_products = user_product[user_product.user_id == u]['product_id']
    # extract avg_reorder_num
    u_reorder = avg_reorder_est[avg_reorder_est.user_id == u]['avg_reorder_num']
    # extract products' topic
    p_topics = topic_product[topic_product.product_id.isin(set(u_products.values[0]))]
    p_topics_pid = p_topics['product_id'].reset_index()['product_id']
    p_topics_vec = p_topics[["topic_%d"%x for x in range(10)]]
    # nbr search, expand search scope
    n_neighbors = 1 * int(np.ceil(u_reorder.values[0]))
    if n_neighbors > len(p_topics_vec):
        n_neighbors = len(p_topics_vec)
    if n_neighbors > 0:
        nbrs = NearestNeighbors(n_neighbors=n_neighbors, 
                                metric = 'l1',
                                algorithm = 'brute').fit(p_topics_vec.values)
        #print(u)
        distances, indices = nbrs.kneighbors(u_topic.values.reshape(1, -1))
        #print(cnt)
        u_nnp.append(list(p_topics_pid[indices[0]].values))
    else:
        u_nnp.append('None')
    

"""
Explanation: 基于方案2的Nearest Neighbors Search

每个人一个NN，找到与user topic最近的商品
分析pred中None的数量
avg_reorder_len, component无normalize, 欧氏聚类时的得分：f1score:0.078288109250376284
avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.11
2 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.15
3 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.178
4 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.19
5 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.20
avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.11:0.107
norm, 2 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.146
norm, 3 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.168
norm, 4 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.18
norm, 5 * avg_reorder_len, component normalize，l1距离时的扥分：f1score:0.19

avg_reorder_len, component normalize，欧氏距离时的扥分：f1score:0.11


avg_reorder_len, component normalize, 对称KL距离时的得分: f1score:0.108
End of explanation
"""


train_pred = pd.DataFrame({'user_id':train_orders.user_id, 
                                 'reorder_pids':u_nnp, 
                                 'order_id':train_orders.order_id})

train_gold = order_products_train[order_products_train.reordered == 1].groupby(['order_id'])['product_id'].apply(list)

train_gold = train_gold.reset_index()

train_eval = pd.merge(train_gold, train_pred, on = ['order_id'], how = 'outer').fillna('None')

train_eval.columns = ['order_id', 'gold_reorder', 'pred_reorder', 'user_id']
    

# 21mins
train_eval['f1score'] = train_eval.apply(wrap_cal_f1, axis = 1)

train_eval['precision'] = train_eval.apply(wrap_cal_precision, axis = 1)

train_eval['recall'] = train_eval.apply(wrap_cal_recall, axis = 1)

train_eval.f1score.mean()

train_eval.describe()

train_eval[train_eval.user_id == 201]

prior_all = pd.merge(order_products_prior, orders, on = ['order_id'], how = 'left')

prior_all = pd.merge(prior_all,
                     product_ad[['product_id', 'product_name', 'aisle', 'department']],
                     on = ['product_id'],
                     how = 'left')

"""
Explanation: 方案2 结果评估
End of explanation
"""


%%time
order_topic = lda_10.transform(order_tf_matrix)

with open(DATA_DIR + 'tf_matrix', 'rb') as f:
    order_tf_matrix = pickle.load(f)

order_products_matrix = order_products_matrix.reset_index()

order_topic = pd.DataFrame(order_topic, columns = ["topic_%d"%x for x in range(10)])

"""
Explanation: 方案1 每张订单的Embeded Topic Space表达
End of explanation
"""


