# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn as sk
import datetime
import seaborn as sns
from datetime import date

train = pd.read_csv("Data/train_users_2.csv")
test = pd.read_csv("Data/test_users.csv")

print('the columns name of training dataset:\n',train.columns)
print('the columns name of test dataset:\n',test.columns)

# +
dac_train = train.date_account_created.value_counts()
dac_test = test.date_account_created.value_counts()
# change data to datetime type
dac_train_date = pd.to_datetime(train.date_account_created.value_counts().index)
dac_test_date = pd.to_datetime(test.date_account_created.value_counts().index)
# Calculate the number of days from the registration time of the first registered user in the training set
dac_train_day = dac_train_date - dac_train_date.min()
dac_test_day = dac_test_date - dac_train_date.min()

plt.scatter(dac_train_day.days, dac_train.values, color = 'r', label = 'train dataset')
plt.scatter(dac_test_day.days, dac_test.values, color = 'b', label = 'test dataset')

plt.title("Accounts created vs day")
plt.xlabel("Days")
plt.ylabel("Accounts created")
plt.legend(loc = 'upper left')
# -

tfa_train_dt = train.timestamp_first_active.astype(str).apply(lambda x:  
                                                                    datetime.datetime(int(x[:4]),
                                                                                      int(x[4:6]), 
                                                                                      int(x[6:8]), 
                                                                                      int(x[8:10]), 
                                                                                      int(x[10:12]),
                                                                                      int(x[12:])))
print(tfa_train_dt.describe())

print(train.date_first_booking.describe())
print(test.date_first_booking.describe())

print(train.age.value_counts().head())

# +
# Divide age into 4 groups,missing values, too small age, reasonable age, too large age
age_train =[train[train.age.isnull()].age.shape[0],
            train.query('age < 15').age.shape[0],
            train.query('age <= 15 & age <25').age.shape[0],
            train.query('age <= 25 & age <35').age.shape[0],
            train.query('age <= 35 & age <45').age.shape[0],
            train.query('age <= 45 & age <55').age.shape[0],
            train.query("age <= 55 & age <65").age.shape[0],
            train.query("age <= 65 & age <75").age.shape[0],
            train.query("age <= 75 & age <85").age.shape[0],
            train.query('age > 85').age.shape[0]]

age_test = [test[test.age.isnull()].age.shape[0],
            test.query('age < 15').age.shape[0],
            test.query("age >= 15 & age <= 85").age.shape[0],
            test.query('age > 90').age.shape[0]]

columns = ['Null', 'age < 15', 'age', 'age > 85']

# plot
fig, (ax1,ax2) = plt.subplots(1,2,sharex=True, sharey = True,figsize=(10,5))

sns.barplot(columns, age_train, ax = ax1)
sns.barplot(columns, age_test, ax = ax2)

ax1.set_title('training dataset')
ax2.set_title('test dataset')
ax1.set_ylabel('counts')


# -

def feature_barplot(feature, df_train = train, df_test = test, figsize=(10,5), rot = 90, saveimg = True): 
    feat_train = df_train[feature].value_counts()
    feat_test = df_test[feature].value_counts()
    fig_feature, (axis1,axis2) = plt.subplots(1,2,sharex=True, sharey = True, figsize = figsize)
    sns.barplot(feat_train.index.values, feat_train.values, ax = axis1)
    sns.barplot(feat_test.index.values, feat_test.values, ax = axis2)
    axis1.set_xticklabels(axis1.xaxis.get_majorticklabels(), rotation = rot)
    axis2.set_xticklabels(axis1.xaxis.get_majorticklabels(), rotation = rot)
    axis1.set_title(feature + ' of training dataset')
    axis2.set_title(feature + ' of test dataset')
    axis1.set_ylabel('Counts')
    plt.tight_layout()
    if saveimg == True:
        figname = feature + ".png"
        fig_feature.savefig(figname, dpi = 75)


feature_barplot('gender', saveimg = True)

feature_barplot('signup_method')

feature_barplot('signup_flow')

feature_barplot('language')

feature_barplot('affiliate_channel')

feature_barplot('first_affiliate_tracked')

feature_barplot('signup_app')

feature_barplot('first_device_type')

feature_barplot('first_browser')


def label_barplot(feature, df_train = train, figsize=(10,5), rot = 90, saveimg = False): 
    feat_train = df_train[feature].value_counts()
    fig = plt.figure(figsize=(8,4))
    sns.barplot(feat_train.index.values, feat_train.values)
    plt.title(feature + ' of training dataset')
    plt.ylabel('Counts')
    plt.tight_layout()
    if saveimg == True:
        figname = feature + ".png"
        fig_feature.savefig(figname, dpi = 75)


label_barplot('country_destination')



StarcoderdataJupyterScriptsDedupFiltered