"""
# Importing Libraries
"""
!pip install pyforest
from pyforest import * 
from sklearn.ensemble import RandomForestRegressor
from sklearn import metrics
from sklearn.metrics import mean_absolute_error
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
plt.rcParams['figure.figsize'] = (15.0, 15.0)
plt.style.use('ggplot')
pd.set_option('display.max_columns', None)  
pd.set_option('display.expand_frame_repr', False)
pd.set_option('max_colwidth', -1)
%matplotlib inline

os.listdir('/kaggle/input/olist-dataset')
"""
# Joining Data Sets
"""
# leads dataset
mql = pd.read_csv('../input/olist-dataset/olist_marketing_qualified_leads_dataset.csv')
mql.head(10)
# closed deals dataset
cd = pd.read_csv('../input/olist-dataset/olist_closed_deals_dataset.csv')
cd.head(10)
# marketing funnel dataset (NaNs are leads that did not close a deal)
mf = mql.merge(cd, on='mql_id', how='left')
mf.head(10)
os.chdir('/kaggle/input/brazilian-ecommerce/')
# sellers dataset
#../input/olist-dataset/olist_marketing_qualified_leads_dataset.csv
sellers=pd.read_csv('olist_sellers_dataset.csv')#/')
sellers.head(10)
# marketing funnel merged with sellers (this way you get seller location)
mf_sellers = mf.merge(sellers, on='seller_id', how='left')
mf_sellers.head(10)
# order items dataset
order_items = pd.read_csv("olist_order_items_dataset.csv")
order_items.head(10)
# marketing funnel merged with items (this way you get products sold by sellers)
mf_items = mf.merge(order_items, on='seller_id', how='left')
mf_items.head(10)
###Getting an Order level Quantity and overall order price
order_payments = pd.read_csv("olist_order_payments_dataset.csv")
orderid_amount=order_payments.groupby(['order_id']).agg({'payment_value':sum,'order_id':len})
orderid_amount.columns=['totalpayment_value','total_qty']
orderid_amount=orderid_amount.reset_index()

customers = pd.read_csv("olist_customers_dataset.csv")
orders = pd.read_csv("olist_orders_dataset.csv")
products=pd.read_csv("olist_products_dataset.csv")
review=pd.read_csv("olist_order_reviews_dataset.csv")
ops=orderid_amount.merge(order_items[['order_id', 'product_id', 'seller_id']],how='left',on='order_id')
display(ops.shape)

ops=ops.merge(products[['product_id','product_category_name','product_photos_qty']],how='left',on=['product_id'])
display(ops.shape)

ops=ops.merge(review[['review_score','order_id']],how='inner',on='order_id')
display(ops.shape)

for col in['order_delivered_customer_date','order_purchase_timestamp']:
    orders[col] = pd.to_datetime(orders[col])
orders['delivery_days']=(orders['order_delivered_customer_date'].dt.date-orders['order_purchase_timestamp'].dt.date).dt.days
ops=ops.merge(orders[['order_id', 'customer_id','order_purchase_timestamp','delivery_days']],how='inner',on='order_id')
display(ops.shape)

ops=ops.merge(customers[['customer_id','customer_city','customer_state']],how='inner',on='customer_id')
display(ops.shape)
ops['order_purchase_month_year'] = pd.to_datetime(ops['order_purchase_timestamp']).dt.to_period('M')
ops.drop(columns=['order_purchase_timestamp'],inplace=True)
ops=ops[ops.seller_id.notnull()]
###Aggregating on an orderid level
ops['rc']=ops.groupby(['order_id']).cumcount()+1
ops=ops[ops.rc==1]
ops.drop(columns={'rc'},inplace=True)
ops.order_purchase_month_year.value_counts()
##Ensuring Jan 2017-Sep 2018 is being considered
ops=ops[(ops.order_purchase_month_year>='2017-01')&(ops.order_purchase_month_year<'2018-10')]
ops.order_purchase_month_year.value_counts()
###Unique on Order_id level
ops.shape,ops.order_id.nunique()
"""
# Life Time Value
"""
ltv=pd.DataFrame(ops.groupby(['seller_id'])['totalpayment_value'].sum()).reset_index()
ltv.describe()
ltv_seller=ltv.quantile(np.linspace(.1, 1.1, 10,0)).reset_index()
ltv_seller.columns=['percentile','ltv']
ltv_seller
"""
# Monthly Revenue
"""
mr=pd.DataFrame(ops.order_purchase_month_year.value_counts()).reset_index()
mr=ops.groupby(['order_purchase_month_year']).agg({'order_id':len,'totalpayment_value':sum}).reset_index()
mr.columns=['year_month','order_count','revenue_generated']
mr.year_month=mr.year_month.astype(str)
mr
# create figure and axis objects with subplots()
fig,ax = plt.subplots()
plt.xticks(rotation=-45)
# make a plot
ax.plot(mr.year_month, mr.order_count, color="red", marker="o")
# set x-axis label
ax.set_xlabel("year_month",fontsize=14)
# set y-axis label
ax.set_ylabel("order_count",color="red",fontsize=14)
ax.set_title('Monthly Performance Jan 2017-Sep 2018')
# twin object for two different y-axis on the sample plot
ax2=ax.twinx()
# make a plot with different y-axis using second axis object
ax2.plot(mr.year_month, mr.revenue_generated,color="blue",marker="o")
ax2.set_ylabel("revenue_generated",color="blue",fontsize=14)
fig.set_size_inches(10,6)
plt.show()




"""
# Top 15 selling product categories
"""
top15_prodcat=pd.DataFrame(ops.product_category_name.value_counts().sort_values(ascending=False).head(15)).reset_index()
top15_prodcat.columns=['product_category_name','order_count']
fig=plt.gcf()
fig.set_size_inches(20,10)
graph=sns.barplot('product_category_name','order_count',data=top15_prodcat)
for p in graph.patches:
        graph.annotate('{:.0f}'.format(p.get_height()), (p.get_x()+0.3, p.get_height()),
                    ha='center', va='bottom',
                    color= 'black')
plt.xticks(rotation=-45)



ops1=ops.copy()
final_data=pd.merge(ops1[ops1.seller_id.notnull()],mf[['seller_id','landing_page_id', 'origin','business_segment',
       'lead_type', 'lead_behaviour_profile','business_type']],how='left',on='seller_id')
final_data.order_purchase_month_year.value_counts()
final_data['month_purchase']=final_data.order_purchase_month_year.dt.month
display(final_data.order_id.nunique(),final_data.shape)
display(final_data.head())
"""
# Data Preprocessing
"""
##dropping high missing value column,specific rows and missing value imputation
for col in final_data.columns:
    if ((final_data[col].isnull().sum())/(final_data.shape[0])>0.95):
        final_data.drop(columns=[col],inplace=True)
        
final_data=final_data[final_data.product_category_name.notnull()]
final_data.delivery_days.fillna(final_data.delivery_days.median(),inplace=True)
final_data.columns
column_list = ['product_category_name','customer_city','customer_state']#,'lead_type','lead_behaviour_profile','business_type','landing_page_id','origin']

for column in column_list:
    encoder = preprocessing.LabelEncoder()
    encoder.fit(final_data[column])
    final_data[column] = encoder.transform(final_data[column])
"""
# Splitting Data into Train and Test samples
"""

features = ['total_qty', 'product_photos_qty','review_score','product_category_name',
             'delivery_days', 'customer_city',
       'customer_state', 'month_purchase']#
target = ['totalpayment_value']

X = final_data[features].values
y = final_data[target].values
split_test_size = 0.10

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = split_test_size, random_state=8)

print('Training Features Shape:', X_train.shape)
print('Testing Features Shape:', X_test.shape)
print('Training labels Shape:', y_train.shape)
print('Testing labels Shape:', y_test.shape)
"""
# Predicting revenue using Random Forest Regressor
"""

rf_model_finaldata = RandomForestRegressor(n_estimators=100, min_samples_split=2, random_state = 20,n_jobs=-1)
rf_model_finaldata.fit(X_train, np.ravel(y_train,order='C'))

rf_predict_train = rf_model_finaldata.predict(X_train)
rf_predict_test = rf_model_finaldata.predict(X_test)

print("Mean Absolute error in predicting train data",mean_absolute_error(y_train,rf_predict_train))
print("Mean Absolute error in predicting test data",mean_absolute_error(y_test,rf_predict_test))
imp_features = pd.Series(rf_model_finaldata.feature_importances_,index=features).sort_values(ascending=False)
imp_features=pd.DataFrame(imp_features.reset_index())
imp_features.columns=['variable_name','importance']
imp_features
"""
# Plotting the Important Features
"""
sns.barplot('variable_name','importance',data=imp_features)
plt.title('Feature Importance Plot')
plt.xticks(rotation=60)

"""
# Correlation Plot
"""
sns.heatmap(final_data.corr())