# Import required libraries(必要ライブラリのインポート)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
import pprint
from sklearn.metrics import silhouette_score
from sklearn.metrics import silhouette_samples
from matplotlib import cm
# Read data(データの読み込み)
!kaggle datasets download -d imakash3011/customer-personality-analysis
df_customer = pd.read_csv('../input/customer-personality-analysis/marketing_campaign.csv', sep='\t')
df_customer.head(15)
"""
# Data Preprocessing(データの前処理)
"""
# Check the data(データ数の確認)
df_customer.shape
"""
ファイルを読み込むときにsepメソッドをしてしないと中身が分割しなかった
 ←元のデータのスタイルがテーブル形式だったのか??


"""
# Check the data type & missing value of each column(各列のデータタイプ&欠損値を確認)
df_customer.info()
# Delete missing data rows(欠損データ行の削除)
df_customer = df_customer.dropna().reset_index(drop = True)
df_customer.head(15)
# Basic Statistics(基本統計量)
df_customer.describe()
# adding new columns(新しいデータ列の追加)
df_customer['Age'] = 2021 - df_customer['Year_Birth']
df_customer['MntTotalProducts'] = df_customer['MntWines'] + df_customer['MntFruits'] + df_customer['MntMeatProducts'] + df_customer['MntFishProducts'] + df_customer['MntSweetProducts'] + df_customer['MntGoldProds']

# Renaming columns(列名の再定義)
numeric_column_all = ['income', 'kidhome',
                   'teenhome', 'recency', 'mntwines', 'mntfruits',
                   'mntmeatproducts', 'mntfishproducts', 'mntsweetproducts',
                   'mntgoldprods', 'numdealspurchases', 'numwebpurchases',
                   'numcatalogpurchases', 'numstorepurchases', 'numwebvisitsmonth',
                   'acceptedcmp3', 'acceptedcmp4', 'acceptedcmp5', 'acceptedcmp1',
                   'acceptedcmp2', 'complain', 'z_costcontact', 'z_revenue', 'response']

numeric_columns = ['income', 'kidhome',
                   'teenhome', 'recency', 'mntwines', 'mntfruits',
                   'mntmeatproducts', 'mntfishproducts', 'mntsweetproducts',
                   'mntgoldprods', 'numdealspurchases', 'numwebpurchases',
                   'numcatalogpurchases', 'numstorepurchases', 'numwebvisitsmonth']

bool_columns = ['acceptedcmp3', 'acceptedcmp4', 'acceptedcmp5', 'acceptedcmp1',
                   'acceptedcmp2', 'complain', 'response']
df_customer = df_customer.rename(columns={'Response' : 'AcceptedCmp6'})

categorical_columns = ['education', 'marital_status']

date_columns = ['year_birth','dt_customer'] 

# Changing data types and dropping columns(データ型の変更&不必要データ列の削除)
df_customer['Dt_Customer'] = pd.to_datetime(df_customer['Dt_Customer'])
df_customer['Education'], df_customer['Marital_Status'] = df_customer['Education'].astype('category'), df_customer['Marital_Status'].astype('category')

df_customer.drop(['Z_CostContact', 'Z_Revenue'], axis=1, inplace=True)

# Classifying DataFrame(特定用途のデータフレームを定義)
variable_all_df = df_customer.loc[ : , ['Age', 'Income', 'Kidhome',
       'Teenhome', 'Recency', 'MntTotalProducts', 'MntWines', 'MntFruits',
       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',
       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',
       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth']]

variable_df = df_customer.loc[ : , ['Year_Birth', 'Education', 'Marital_Status', 'Income', 'Kidhome',
       'Teenhome', 'Dt_Customer', 'Recency']]

mnt_df = df_customer.loc[ : , ['MntTotalProducts', 'MntWines', 'MntFruits',
       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',
       'MntGoldProds']]
num_purchases_df = df_customer.loc[ : , ['NumDealsPurchases', 'NumWebPurchases',
       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth']]

campaign_df = df_customer.loc[ : , ['AcceptedCmp1', 'AcceptedCmp2', 'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp6']]


"""
### Confirm data distribution(データの分布確認)
"""
# Function definition of box plot(箱ヒゲ図の関数定義)
def get_box(input_data):
  output_data = input_data.copy()
  fig = plt.figure(figsize=(20,100))
  num_columns = output_data.columns
  for i in range(len(num_columns)):
    plt.subplot(len(num_columns), 4, i+1)
    output_data[num_columns[i]].plot(kind = "box")
  return output_data
# illustrate the box plot of each row(各列の箱ヒゲ図を図示)
df_var = get_box(variable_all_df)
"""
Kidhome, Teenhome, Recency, NumStorePurchasesの4つ以外の列に外れ値が存在する事が確認できた
"""
"""
### Detect outliers using IQR(IQRを用いて外れ値を検出)
"""
# Define function of IQR(IQR関数を定義)
def Outlier_By_IQR(input_data):
  output_data = input_data.copy()
  numeric_columns_to_filter = ['Age', 'Income','MntTotalProducts', 'MntWines', 'MntFruits',
       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',
       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',
       'NumCatalogPurchases', 'NumWebVisitsMonth']
  for i in range(len(numeric_columns_to_filter)):
    q1 = output_data[numeric_columns_to_filter[i]].quantile(0.25)
    q3 = output_data[numeric_columns_to_filter[i]].quantile(0.75)
    iqr = q3 - q1
    bottom = q1 - 1.5*iqr
    up = q3 + 1.5*iqr
    print(numeric_columns_to_filter[i])
    print(f'Q1：{q1}')
    print(f'Q3：{q3}')
    print(f'IQR：{iqr}')
    print('Outlier is... 外れ値は↓')
    print(output_data[numeric_columns_to_filter[i]][(output_data[numeric_columns_to_filter[i]] < bottom) | (output_data[numeric_columns_to_filter[i]] > up)])
    print('********************************************************')
  return output_data

# Execute function of IQR(IQR関数を実行)
df_outlier = Outlier_By_IQR(variable_all_df)
"""
購入金額や購入経路に関しては明確な外れ値は存在しない。
一方で年齢や収入に関しては現実的でない値や明らかに標準偏差を歪ませているデータがあるため、外れ値として排除する。
"""
# Deleting outliers(外れ値を除去)
numeric_columns_to_filter = ['Age', 'Income']

Q1 = df_customer[numeric_columns_to_filter].quantile(0.25)
Q3 = df_customer[numeric_columns_to_filter].quantile(0.75)
IQR = Q3 - Q1

df_filtered = df_customer[~((df_customer[numeric_columns_to_filter] < (Q1 - 1.5 * IQR)) |(df_customer[numeric_columns_to_filter] > (Q3 + 1.5 * IQR))).any(axis=1)]

display(df_customer.shape)
display(df_filtered.shape)
"""
# EDA(Explanatory Data Analysis) 
"""
# Scatter plot of purchaser characteristics <sorted by educational background>(購入者の特徴に関する散布図＜学歴別にソート＞)
sns.pairplot(data=variable_df, hue='Education')
# Scatter plot of buyer characteristics <sorted by marriage status>(購入者の特徴に関する散布図＜結婚状況別にソート＞）
sns.pairplot(data=variable_df, hue='Marital_Status')
# Scatter plot of purchase amount by item(品目別購入額の散布図)
sns.pairplot(data=mnt_df)
# Scatter plot of purchase route(購入経路の散布図)
sns.pairplot(data=num_purchases_df)
# Defined by counting the number of elements in each age group(各年齢層の要素数をカウントして定義)
age_teenager = len(df_customer[(df_customer['Age']>0)&(df_customer['Age']<20)])
age_20s = len(df_customer[(df_customer['Age']>=20)&(df_customer['Age']<30)])
age_30s = len(df_customer[(df_customer['Age']>=30)&(df_customer['Age']<40)])
age_40s = len(df_customer[(df_customer['Age']>=40)&(df_customer['Age']<50)])
age_50s = len(df_customer[(df_customer['Age']>=50)&(df_customer['Age']<60)])
age_over60 = len(df_customer[(df_customer['Age']>=60)])

df_customer['Age Group'] = pd.Series()

for i in range(len(df_customer['Age'])):
  if (df_customer.at[i, 'Age']<30):
    df_customer.at[i, 'Age Group'] = 20
  elif (df_customer.at[i, 'Age']<40):
    df_customer.at[i, 'Age Group'] = 30
  elif (df_customer.at[i, 'Age']<50):
    df_customer.at[i, 'Age Group'] = 40
  elif (df_customer.at[i, 'Age']<60):
    df_customer.at[i, 'Age Group'] = 50
  else:
    df_customer.at[i, 'Age Group'] = 60

# Save the count number as a dictionary by defining the column name(カウント数をコラム名を定義して辞書型で保存)
l1 = ["teenager", "20s", "30s", "40s", "50s", "over60"]
l2 = [age_teenager, age_20s, age_30s, age_40s, age_50s, age_over60]

age_df = pd.DataFrame({'age group' : l1, 'count' : l2})

age_df
# Illustration of age group with bar graph(年齢層を棒グラフで図示)

fig, ax = plt.subplots(figsize = (10, 4.8))
plt.barh(age_df['age group'], age_df['count'], color = 'forestgreen')
plt.title('Count of Customers by Age Group')
plt.xlabel('Age Group')
plt.ylabel('Count')
# Cumulative by educational background(学歴別に累計)
education_columns = df_customer['Education'].unique()
dataset_education = df_customer['Education'].tolist()

edu_item_count = []

for edu_column in education_columns:
  edu_count = dataset_education.count(edu_column)
  print(f'{edu_column}：{edu_count}')

  edu_item_count.append(edu_count)
# A bar graph showing the cumulative total by educational background(学歴別累計を棒グラフで図示)
y = education_columns
width = edu_item_count

fig, ax = plt.subplots(figsize = (10, 4.8))
plt.barh(y=y, width=width)
plt.title('Count of Customers by Education')
plt.xlabel('Count')
plt.ylabel('Education')
# Cumulative by marital status(結婚状況別に累計)
marital_columns = df_customer['Marital_Status'].unique()
df_customer_marital = df_customer['Marital_Status'].tolist()

mar_item_count = []

for mar_col in marital_columns:
  mar_count = df_customer_marital.count(mar_col)
  print(f'{mar_col}：{mar_count}')

  mar_item_count.append(mar_count)
# A bar graph showing the cumulative total by marital status(結婚状況別累計を棒グラフで図示)
y = marital_columns
width = mar_item_count
fig, ax = plt.subplots(figsize = (10, 4.8))
plt.barh(y=y, width=width, color='pink')
plt.xlabel('Count')
plt.ylabel('Marital Status')
# Classification by income status(収入状況で分類)
df_filtered['Income'].sort_values(ascending = False)

# Illustrate the distribution map according to the income situation(収入状況でヒストグラムを図示)
fig, ax = plt.subplots(figsize = (8.0, 4.8))
sns.distplot(df_filtered['Income'])
plt.title('Income Distribution')
plt.xlabel('Income($)')
plt.ylabel('Count')
# Illustration of purchase frequency in a bar graph
recency = df_customer['Recency'].value_counts()
plt.bar(recency.index, recency.values)
# A bar graph showing the distribution of responses to marketing campaigns(マーケティングキャンペーンへの反応の分布を棒グラフで図示)
for i in range(1, 7):
  campaign_df[f'AcceptedCmp{i}'] = campaign_df[f'AcceptedCmp{i}'].astype(str)
campaign_df['AcceptedCmp_all'] = campaign_df['AcceptedCmp1'] + campaign_df['AcceptedCmp2'] + campaign_df['AcceptedCmp3'] + campaign_df['AcceptedCmp4'] + campaign_df['AcceptedCmp5'] + campaign_df['AcceptedCmp6']
campaign_df['AcceptedCmp_all'].astype(int)

x = campaign_df['AcceptedCmp_all'].unique()
x = np.sort(x)
y = campaign_df['AcceptedCmp_all'].value_counts()
y = y.sort_index()

fig, ax = plt.subplots(figsize=(10.0, 6.0))
plt.bar(x=x, height=y)
plt.title('Distribution of Response to each Campaign')
plt.xlabel('each Campaign')
plt.ylabel('Count')
plt.xticks(rotation=90)
"""
# VS
"""
"""
## Changes in EC subscribers(EC加入者の推移)
"""
#  Temporarily change the index to Datetime(インデックスを一時的に変更)
df_customer.set_index('Dt_Customer', inplace=True)

date_enrollment = df_customer['ID'].resample('M').count()

values_date_enrollment = date_enrollment.tolist()
sigma_values_date_enrollment = []
for i in range(len((values_date_enrollment))):
  sigma_values_date_enrollment.append(sum(values_date_enrollment[:i]))
index_date_enrollment = date_enrollment.index

fig, ax = plt.subplots(figsize = (10.0, 4.8))
plt.plot(index_date_enrollment, sigma_values_date_enrollment, marker = 'o')
plt.title('Transition of Subscriber')
plt.xlabel('Date')
plt.ylabel ('Count of Enrollment')
plt.show()

# Redefine df_customer index(# df_customerのインデックスを再定義)
df_customer = df_customer.reset_index()
"""
Note that the Dt_Customer column comes before the ID column
ID列の前にDt_Customer列が来ていることに注意
"""
df_customer.head()
"""
## About the user's maximum purchase item(ユーザーの最大購入品目について）
"""
# Convert purchased product columns to numbers and determine maximum for each row(購入製品列を数字に変換し、各行の最大を判定)

df_customer['1st Best Product'] = pd.Series()
df_customer['2nd Best Product'] = pd.Series()
best_prod = df_customer.iloc[ : , 10:15]
best_prod = best_prod.rename(columns = {'MntWines' : 1, 'MntFruits' : 2, 'MntMeatProducts' : 3, 'MntFishProducts' : 4, 'MntSweetProducts' : 5, 'MntGoldProds' : 6})

for i in range(len(best_prod)):
  d = dict(best_prod.iloc[i])

  m = max(d, key=d.get)
  df_customer['1st Best Product'].iloc[i] = int(m)

  del d[m]
  df_customer['2nd Best Product'].iloc[i] = int(max(d, key = d.get))

  df_customer.head()
# Scatter plot <total purchase vs income>(散布図＜購入総額と収入＞)
fig, ax = plt.subplots(figsize = (8.0, 4.8))
plt.scatter(df_filtered['MntTotalProducts'], df_filtered['Income'])
plt.title('MntTotalProducts vs Income')
plt.ylabel('Income ($)')
plt.xlabel('Mount of Total Products ($)')
# Scatter plot <total purchase amount vs frequency of purchase>(散布図＜購入総額と購入頻度＞)
fig, ax = plt.subplots(figsize = (8.0, 4.8))
plt.scatter(df_filtered['Recency'], df_filtered['MntTotalProducts'])
plt.title('Recency vs MntTotalProducts')
plt.xlabel('Last Days (days)')
plt.ylabel('Mount of Total Products ($)')
"""
# Clustering analysis(クラスタリング分析）
"""
# Check the correlation of the entire data(全体データの相関関係を確認)
plt.figure(figsize=(20,20))
sns.heatmap(df_filtered.corr(), annot=True)
"""
MntTotalProducts列はMnt〇〇列の総和でありこれらとの相関係数が極端に高い事からクラスタリングに用いるデータとしてはノイズと判断し、除去しておく
"""
# Store explanatory variables in a data frame(説明変数をデータフレームに格納)
df_clustering = df_filtered.loc[ : , ['Age', 'Income', 'Kidhome', 'Teenhome', 'Recency',
       'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts',
       'MntSweetProducts', 'MntGoldProds', 'NumDealsPurchases',
       'NumWebPurchases', 'NumCatalogPurchases', 'NumStorePurchases',
       'NumWebVisitsMonth']]

df_clustering.head()
# Data standardization(データの標準化)
scaler = StandardScaler()
clustering_scalered = scaler.fit_transform(df_clustering)
"""
## Apply K-means method(K-平均法を適応)
"""
# Difine K-means method(K-平均法を定義)
kmeans_kwargs = {
    "init" : "random",
    "n_init" : 10,
    "max_iter" : 300,
    "random_state" : 42
}

sse = []
for k in range(1, 11):
  kmeans = KMeans(n_clusters=k, **kmeans_kwargs)
  kmeans.fit(clustering_scalered)
  sse.append(kmeans.inertia_)

# Illustrated elbow method(エルボー法を図示)
plt.style.use("fivethirtyeight")
plt.plot(range(1, 11), sse, marker = "o")
plt.xticks(range(1, 11))
plt.xlabel('Number of Cluster')
plt.ylabel('Distortion(SSE)')
plt.title('Kmeans Number of Clustering')
plt.show()
"""
上記の表（エルボー図）からクラス多数は2もしくは3程度が妥当だと考えられる。
"""
# Run with 3 clusters(クラスター数を3で実行)
kmeans = KMeans(3)
kmeans.fit(df_clustering)
identified_clusters = kmeans.fit_predict(df_clustering)


s = set(identified_clusters)
l = list(s)

print(l)
print('*************************************************')
identified_clusters
# Check the centroid coordinates for each cluster are stored(各クラスターのセントロイドの座標が格納されていることを確認)
kmeans.cluster_centers_
# Add a cluster to a data frame(クラスターをデータフレームに追加)
df_clustering['Cluster'] = identified_clusters
df_clustering.head()
"""
## Validate the number of clusters by silhouette analysis(シルエット分析でクラスター数の妥当性を確認)
"""
# Performing silhouette analysis and illustration in bar chart(シルエット分析の実行と横棒グラフでの図示)
"""
* A scatter plot of each cluster is also illustrated when there is time
* 時間があるときに各クラスタの散布図も図示する
"""

score = silhouette_score(clustering_scalered, kmeans.labels_)
print(round(score, 4))

silhouette_values = silhouette_samples(df_clustering, identified_clusters, metric='euclidean')
y_ax_lower, y_ax_upper = 0, 0
y_ticks = []
n_clusters = len(l)

# Standardization of the number of clusters(クラスター数の標準化)
np_l = np.array(l)
np_1 = np.array([1,1,1])
l_std = np_l + np_1

for i, c in enumerate(l):
  c_silhouette_vals = silhouette_values[identified_clusters == c]
  c_silhouette_vals.sort()
  y_ax_upper += len(c_silhouette_vals)
  color = cm.jet(float(i) / n_clusters)
  plt.barh(range(y_ax_lower, y_ax_upper),
           c_silhouette_vals,
           height = 1.0,
           edgecolor = 'none',
           color = color)
  y_ticks.append((y_ax_lower + y_ax_upper) / 2)
  y_ax_lower += len(c_silhouette_vals)

silhouette_avg = np.mean(silhouette_values)
plt.axvline(silhouette_avg, color = 'red', linestyle = '--')
plt.yticks(y_ticks, l_std)
plt.ylabel('Cluster')
plt.xlabel('silhouette coefficient')
plt.show()
# 各クラスターの定義と特徴分析
cluster_0 = df_clustering[df_clustering['Cluster'] == 0]
cluster_1 = df_clustering[df_clustering['Cluster'] == 1]
cluster_2 = df_clustering[df_clustering['Cluster'] == 2]
# クラスター0の特徴
cluster_0_desc = cluster_0.describe().transpose()
cluster_0_desc
# クラスター1の特徴
cluster_1_desc = cluster_1.describe().transpose()
cluster_1_desc
# クラスター2の特徴
cluster_2_desc = cluster_2.describe().transpose()
cluster_2_desc
cluster_index = cluster_0.index

cluster_cmp = pd.DataFrame()
cluster_cmp = cluster_cmp.append(cluster_0_desc['mean'])
cluster_cmp = cluster_cmp.rename(index={'mean' : 'cluster_0'})
cluster_cmp = cluster_cmp.append(cluster_1_desc['mean'])
cluster_cmp = cluster_cmp.rename(index={'mean' : 'cluster_1'})
cluster_cmp = cluster_cmp.append(cluster_2_desc['mean'])
cluster_cmp = cluster_cmp.rename(index={'mean' : 'cluster_2'})
cluster_cmp.T