"""
# 对二手车价格进行回归预测
"""
"""
## Step1: 导入相关模块
"""
import numpy as np
import pandas as pd
import os
import seaborn as sns
"""
## Step2: 导入数据并对数据做简单探索
"""
df = pd.read_csv('../input/craigslist-carstrucks-data/vehicles.csv', index_col=0)
print("data shape:")
print(df.shape)
print("columns name:")
print(list(df.columns))
"""
## Step 3: 数据清洗
"""
"""
### 3.1 删除暂时无法处理的列数据，例如url地址，车辆图像，车辆描述等
"""
useless_columns = ['url', 'region_url', 'VIN', 'image_url', 'description', 'state']
df.drop(useless_columns, axis=1, inplace=True)
print("data shape:")
print(df.shape)
print("columns name:")
print(list(df.columns))
"""
### 3.2 查看数据缺失情况，对关键数据缺失的样本进行剔除
"""
df.isnull().any(axis=0)
temp = df.dropna(axis=0, subset=['year', 'manufacturer'])
print("共删除", df.shape[0]-temp.shape[0], "行")
df = temp
"""
### 3.3 对无效数据>6个的进行删除
"""
def count_invalid_num(series):
    """
    查找无效数据，返回无效数据个数
    """
    invalid_num = series.isnull().sum()
    if series.price <= 0:
        invalid_num += 1
    if series.odometer < 0:
        invalid_num += 1
    return invalid_num

df['invalid_num'] = df.apply(count_invalid_num, axis=1)
df.head()
ori_rows = df.shape[0]
df.drop(df[df.invalid_num > 6].index, inplace=True)
after_rows = df.shape[0]
print("共删除", ori_rows - after_rows, "行")
"""
### 3.4 对price=0的行进行删除
"""
ori_rows = df.shape[0]
df.drop(df[df['price'] == 0].index, inplace = True) 
after_rows = df.shape[0]
print("共删除", ori_rows - after_rows, "行")
print("目前数据形状为：", df.shape)
"""
## Step4: 数据插补
"""
"""
### 准备插补函数
"""
def fill_by_key(data:pd.DataFrame, tar_col:str, type:str, key:str, fill=False, default='GT'):
    if type == 'median':
        tmp = dict(data.groupby(key)[tar_col].median())
    if type == 'mode':
        tmp = dict(df.groupby(key)[tar_col].agg(lambda x: pd.Series.mode(x)))
        for (k, v) in tmp.items():
            if str(v).find('[') != -1:
                print(k, v, '->', default)
                tmp[k] = default
    if type == 'mean':
        tmp = dict(df.groupby(key)[tar_col].mean())
    if fill:
        df[tar_col] = df[tar_col].fillna(df[key].apply(lambda x: tmp.get(x)))
        df.drop(df[df[tar_col].isna()].index, inplace = True)
    else:
        return tmp
    
def fill_helper(data:pd.DataFrame, diction:dict, tar_col:str, key:str):
    df[tar_col] = df[tar_col].fillna(df[key].apply(lambda x: diction.get(x)))
    df.drop(df[df[tar_col].isna()].index, inplace = True)
"""
### 4.1 对行驶公里数（odometer）进行插补
"""
fill_by_key(df, 'odometer', 'median', 'year', True, 1000)  # 根据year的中位数进行插补
print("目前数据形状为：", df.shape)
df.isnull().any(axis=0)
"""
### 4.2 对车型（model）缺失值进行插补
"""
fill_ = fill_by_key(df, 'model', 'mode', 'manufacturer')
fill_helper(df, fill_, 'model', 'manufacturer')
df[df.manufacturer=='hennessey'].model
df.isnull().any(axis=0)
"""
### 4.3 对车况（condition）进行缺失值插补
"""
df.condition.unique()
def condition2int(condition):
    condition_dict = {'salvage':0, 'fair':1, 'good':2, 'excellent':3, 'like new':4, 'new':5}
    try:
        return condition_dict[condition]
    except:
        return np.nan

df['condition'] = df['condition'].apply(condition2int)
fill_by_key(df, 'condition', 'median', 'year', True, 2)
df.isnull().any(axis=0)
"""
### 4.4 对车轮（cylinders）进行缺失值插补
"""
df.cylinders.unique()
def cylinder2int(cylinders):
    try:
        return int(cylinders[0])
    except:
        return np.nan
df['cylinders'] = df['cylinders'].apply(cylinder2int)
fill_ = fill_by_key(df, 'cylinders', 'median', 'model', default=4)
fill_helper(df, fill_, 'cylinders', 'model')
"""
### 4.5 对燃料形式（fuel）进行插补
"""
fill_mode = fill_by_key(df, 'fuel', 'mode', 'model', default='gas')
fill_helper(df, fill_mode, 'fuel', 'model')
df.isnull().any(axis=0)
"""
### 4.6 对title_status进行插补
"""
df.title_status.unique()
fill_ = fill_by_key(df, 'title_status', 'mode', 'model', default='clean')
fill_helper(df, fill_, 'title_status', 'model')
df.isnull().any(axis=0)
"""
### 4.7 对transmission、drive、size、paint_color进行插补
"""
df.transmission.unique()
fill_ = fill_by_key(df, 'transmission', 'mode', 'model', default='automatic')
fill_helper(df, fill_, 'transmission', 'model')
df.isnull().any(axis=0)
df.drive.unique()
fill_ = fill_by_key(df, 'drive', 'mode', 'model', default='fwd')
fill_helper(df, fill_, 'drive', 'model')
df.isnull().any(axis=0)
df.paint_color.unique()
fill_ = fill_by_key(df, 'paint_color', 'mode', 'model', default='white')
fill_helper(df, fill_, 'paint_color', 'model')
df.isnull().any(axis=0)
df['size'].unique()
fill_ = fill_by_key(df, 'size', 'mode', 'model', default='mid-size')
fill_helper(df, fill_, 'size', 'model')
df.isnull().any(axis=0)
df['type'].unique()
fill_ = fill_by_key(df, 'type', 'mode', 'model', default='other')
fill_helper(df, fill_, 'type', 'model')
df.isnull().any(axis=0)
"""
### 4.8 对经纬度（lat long）进行均值插补
"""
df.long.fillna(df.long.mean(), inplace=True)
df.lat.fillna(df.lat.mean(), inplace=True)
df.isnull().any(axis=0)
"""
## Step 5: 建立回归模型
"""
df.info()
df.condition = df.condition.apply(lambda x: int(x))
df.condition.unique()
df.info()
df.cylinders = df.cylinders.apply(lambda x: int(x))
df.cylinders.unique()
# !pip install catboost 如果没有这个包，则安装
from sklearn.model_selection import train_test_split
from catboost import CatBoostRegressor
X_train, X_val, y_train, y_val = train_test_split(df.loc[:, [x for x in list(df.columns) if x not in ['price', 'id']]], df.loc[:, 'price'], test_size=0.2 , random_state=2021)
print(X_train.shape, X_val.shape, y_train.shape, y_val.shape)
X_train.info()
categorical_features_indices = np.where(X_train.dtypes != np.float)[0]
model = CatBoostRegressor(iterations=1000, depth=5, cat_features=categorical_features_indices,learning_rate=0.05, logging_level='Verbose')
model.fit(X_train, y_train, plot=True)
"""
## Step 6: 预测
"""
y_hat = model.predict(X_val)
import plotly.graph_objects as go
import numpy as np
N = 50
x = np.arange(0,N)
fig = go.Figure()
fig.add_trace(go.Scatter(x=x, y=np.array(y_hat[:N]),mode='lines+markers',name='y_hat'))
fig.add_trace(go.Scatter(x=x, y=np.array(y_val[:N]),mode='lines+markers',name='y_val'))
fig.show()
"""
### 发现价格预测出负数，修改数据，这里可以体现出野值数据对模型的影响
"""
y_train_log, y_val_log = np.log(y_train), np.log(y_val)
model.fit(X_train, y_train_log, plot=True)
y_hat_log = model.predict(X_val)
y_hat = np.exp(y_hat_log)
N = 50
x = np.arange(0,N)
fig = go.Figure()
fig.add_trace(go.Scatter(x=x, y=np.array(y_hat[:N]),mode='lines+markers',name='y_hat'))
fig.add_trace(go.Scatter(x=x, y=np.array(y_val[:N]),mode='lines+markers',name='y_val'))
fig.show()
from sklearn.metrics import mean_squared_error, mean_absolute_error
rmse_val = np.sqrt(mean_squared_error(y_val_log, y_hat_log))
print("RMSE of Validation is: ", rmse_val)