""" # Customer segmentation This notebook aims at analyzing the content of an E-commerce database that lists purchases made by $\sim$4000 customers over a period of one year (from 2010/12/01 to 2011/12/09). Based on this analysis, I develop a model that allows to anticipate the purchases that will be made by a new customer, during the following year and this, from its first purchase. <br> ___ **1. Read and Explore Data** **2. Remove cancelled Orders and Look at Remaining Records** **3. Make Product Categories** **4. Make Customer Categories** **5. Make Customer Value Measure** **6. Examine Item Co-occurence and Make Recommendations** **7. Classification of Customers (under construction)** """ !pip install lifetimes """ # 1. Read and Explore Data """ import pandas as pd from datetime import datetime start_time = datetime.now() # need to install these libraries the first time you run the notebook # %pip install --user -U nltk # %conda install -c conda-forge wordcloud # %conda install -c plotly plotly # %pip install pandas-profiling # %pip install mlxtend # %pip install surprise # %pip install spotlight # %pip install scipy == 1.1.0 # %pip install lifetimes #Because of issues with pandas profiling versions when installed using pip/conda, this code is run to resolve it # %pip install https://github.com/pandas-profiling/pandas-profiling/archive/master.zip import nltk nltk.download('punkt') nltk.download('averaged_perceptron_tagger') nltk.download('stopwords') from nltk.corpus import stopwords from lifetimes import BetaGeoFitter from scipy.special import logsumexp import numpy as np import matplotlib as mpl import matplotlib.pyplot as plt import seaborn as sns import warnings import matplotlib.cm as cm import itertools # import pandas_profiling import plotly.graph_objs as go import math # from spotlight.interactions import Interactions from pathlib import Path from sklearn.preprocessing import StandardScaler from sklearn.cluster import KMeans from sklearn.metrics import silhouette_samples, silhouette_score from sklearn import preprocessing, model_selection, metrics, feature_selection from sklearn.model_selection import GridSearchCV, learning_curve from sklearn.svm import SVC from sklearn.metrics import confusion_matrix from sklearn import neighbors, linear_model, svm, tree, ensemble from scipy.spatial.distance import cdist from wordcloud import WordCloud, STOPWORDS from sklearn.ensemble import AdaBoostClassifier from sklearn.decomposition import PCA from IPython.display import display, HTML from datetime import datetime from plotly.offline import init_notebook_mode,iplot from sklearn.manifold import TSNE from mlxtend.frequent_patterns import apriori from mlxtend.frequent_patterns import association_rules init_notebook_mode(connected=True) warnings.filterwarnings("ignore") # plt.rcParams["patch.force_edgecolor"] = True plt.style.use('fivethirtyeight') mpl.rc('patch', edgecolor = 'dimgray', linewidth=1) %matplotlib inline print('LOAD TIME: ',datetime.now() - start_time) # Load duration about 2 minutes """ Then, I load the data. Once done, I also give some basic informations on the content of the dataframe: the type of the various fields, the number of null values and their percentage with respect to the total number of entries: """ # read the datafile df_initial = pd.read_csv('../input/dataskss/data (7) (1).csv',encoding="ISO-8859-1", dtype={'CustomerID': str,'InvoiceID': str}) print('Dataframe dimensions:', df_initial.shape) #______ df_initial['InvoiceDate'] = pd.to_datetime(df_initial['InvoiceDate']) #____________________________________________________________ # gives some info on columns types and numer of null values tab_info=pd.DataFrame(df_initial.dtypes).T.rename(index={0:'column type'}) tab_info=tab_info.append(pd.DataFrame(df_initial.isnull().sum()).T.rename(index={0:'null values (nb)'})) tab_info=tab_info.append(pd.DataFrame(df_initial.isnull().sum()/df_initial.shape[0]*100).T. rename(index={0:'null values (%)'})) display(tab_info) #__________________ # show first lines display(df_initial[:5]) # print first 5 rows df_initial.head(5) # Output a profile of the original, raw data # profile = pandas_profiling.ProfileReport(df_initial) # profile.to_file('Initial data profile.html') """ While looking at the number of null values in the dataframe, it is interesting to note that $\sim$25% of the entries are not assigned to a particular customer. With the data available, it is impossible to impute values for the user and these entries are thus useless for the current exercise. So I delete them from the dataframe: """ """ This is a lot of orders to just drop/ignore. Before deleting the entries that are not assigned to a particular customer, it is worth taking a look at these entries and explore reasons on why they are not associated with any customers. """ ## Look at entries with no customer ID no_cust = df_initial[df_initial["CustomerID"].isnull() == True] no_cust.head() #print last 5 rows from the dataset no_cust.tail() #count the values no_cust["Country"].value_counts() """ From the exploration, although there is no particular invoice or product associated with no Customer ID, majority of the entry is from the UK. I think dropping the entries without investigating is a rushed decision because one possible reason that these entries don't have an associated Customer ID is because the customer is a new customer (i.e. not assigned an ID yet). I think a more reasonable choice would be to assign an independent Customer ID such as the InvoiceNo (since items with the same InvoiceNo would be assumed to be purchased by the same customer). """ ## Check to make sure that the CustomerIDs don't go as high as the InvoiceNo. pd.DataFrame(df_initial["CustomerID"].value_counts().reset_index().sort_values(by = "index",ascending = False)) pd.DataFrame(df_initial["InvoiceNo"].value_counts().reset_index().sort_values(by = "index",ascending = False)) """ ## Two choice of what to do with the orders with no Customer number. Use one or the other below: """ # 1. study the empty Customer IDs seperately: no_cust # StockCode plt.figure() temp = pd.DataFrame(no_cust["StockCode"].value_counts()) plt.bar(temp.index[:15], temp.StockCode[:15]) plt.xticks(rotation = 90) plt.xlabel("StockCode") plt.ylabel("Count") plt.title("Count of StockCode") plt.show() # 3. Description plt.figure() temp = pd.DataFrame(no_cust["Description"].value_counts()) plt.bar(temp.index[:15], temp.Description[:15]) plt.xticks(rotation = 90) plt.xlabel("Description") plt.ylabel("Count") plt.title("Count of Description") plt.show() # 4. Quantity plt.figure() plt.hist(no_cust["Quantity"], bins = 100) plt.yscale("log") plt.xlabel("Quantity") plt.ylabel("Count") plt.title("Distribution of Quantity") plt.show() # Zoom in on the area between -2000 to 2000 temp = no_cust[(no_cust["Quantity"]<2000) & (no_cust["Quantity"]>-2000)] plt.figure() plt.hist(temp["Quantity"], bins = 100) plt.yscale("log") plt.xlabel("Quantity") plt.ylabel("Count") plt.title("Distribution of Quantity") plt.show() # 5. InvoiceDate plt.figure() temp = pd.DataFrame(no_cust["InvoiceDate"].value_counts()).reset_index() plt.bar(temp["index"][:15].astype(str), temp.InvoiceDate[:15]) plt.xticks(rotation = 90) plt.xlabel("InvoiceDate") plt.ylabel("Count") plt.title("Count of Date") plt.show() # 6. UnitPrice plt.figure() plt.hist(no_cust["UnitPrice"], bins = 100) #plt.xticks(rotation = 90) #plt.xlim(-200,200) plt.yscale("log") plt.xlabel("Price") plt.ylabel("Count") plt.title("Distribution of UnitPrice") plt.show() # Zoom in on the area between 0 to 2000 temp = no_cust[(no_cust["UnitPrice"]<2000) & (no_cust["UnitPrice"]>0)] plt.figure() plt.hist(temp["UnitPrice"], bins = 100) plt.yscale("log") plt.xlabel("Price") plt.ylabel("Count") plt.title("Distribution of UnitPrice") plt.show() # 7. CustomerID plt.figure() temp = pd.DataFrame(no_cust["CustomerID"].value_counts()) plt.bar(temp.index[:15], temp.CustomerID[:15]) plt.xticks(rotation = 90) plt.xlabel("CustomerID") plt.ylabel("Count") plt.title("Count of CustomerID") plt.show() # 8. Country plt.figure() temp = pd.DataFrame(no_cust["Country"].value_counts()) plt.bar(temp.index[:15], temp.Country[:15]) plt.yscale("log") plt.xticks(rotation = 90) plt.xlabel("Country") plt.ylabel("Count") plt.title("Count of Country") plt.show() """ ## Observations * From above we can observe that from more number of customers and orders from United Kingdom country * Least number of customer's are from Bahrain country Compare to all other countries """ # or, 2. alternatively, just drop the orders with no customer ID: df_initial.dropna(axis = 0, subset = ['CustomerID'], inplace = True) print('Dataframe dimensions:', df_initial.shape) df_initial.dropna(axis = 0, subset = ['CustomerID'], inplace = True) print('Dataframe dimensions:', df_initial.shape) #____________________________________________________________ # gives some info on columns types and numer of null values tab_info=pd.DataFrame(df_initial.dtypes).T.rename(index={0:'column type'}) tab_info=tab_info.append(pd.DataFrame(df_initial.isnull().sum()).T.rename(index={0:'null values (nb)'})) tab_info=tab_info.append(pd.DataFrame(df_initial.isnull().sum()/df_initial.shape[0]*100).T. rename(index={0:'null values (%)'})) display(tab_info) """ OK, therefore, by removing these entries we end up with a dataframe filled at 100% for all variables! Finally, I check for duplicate entries and delete them: """ # drop duplicate entries. print('Duplicate entries: {}'.format(df_initial.duplicated().sum())) df_initial.drop_duplicates(inplace = True) """ ___ ## Exploring the content of fields This dataframe contains 8 fields that correspond to: **InvoiceNo**: Invoice number. Nominal, a 6-digit integral number uniquely assigned to each transaction. If this code starts with letter 'c', it indicates a cancellation. <br> **StockCode**: Product (item) code. Nominal, a 5-digit integral number uniquely assigned to each distinct product. <br> **Description**: Product (item) name. Nominal. <br> **Quantity**: The quantities of each product (item) per transaction. Numeric. <br> **InvoiceDate**: Invice Date and time. Numeric, the day and time when each transaction was generated. <br> **UnitPrice**: Unit price. Numeric, Product price per unit in sterling. <br> **CustomerID**: Customer number. Nominal, a 5-digit integral number uniquely assigned to each customer. <br> **Country**: Country name. Nominal, the name of the country where each customer resides.<br> ___ ### Field Statistics """ """ For numeric fields, I look at the distribution of values. For categorical fields, look at the top 15 most occurring values. """ # 1. InvoiceNo. - who made the biggest orders df_initial["InvoiceNo"].value_counts() plt.figure() temp = pd.DataFrame(df_initial["InvoiceNo"].value_counts()) plt.bar(temp.index[:15], temp.InvoiceNo[:15]) plt.xticks(rotation = 90) plt.xlabel("InvoiceNo") plt.ylabel("Count") plt.title("Count of InvoiceNo") plt.show() # StockCode plt.figure() temp = pd.DataFrame(df_initial["StockCode"].value_counts()) plt.bar(temp.index[:15], temp.StockCode[:15]) plt.xticks(rotation = 90) plt.xlabel("StockCode") plt.ylabel("Count") plt.title("Count of StockCode") plt.show() # 3. Description plt.figure() temp = pd.DataFrame(df_initial["Description"].value_counts()) plt.bar(temp.index[:15], temp.Description[:15]) plt.xticks(rotation = 90) plt.xlabel("Description") plt.ylabel("Count") plt.title("Count of Description") plt.show() # 4. Quantity plt.figure() plt.hist(df_initial["Quantity"], bins = 100) plt.yscale("log") plt.xlabel("Quantity") plt.ylabel("Count") plt.title("Distribution of Quantity") plt.show() # Zoom in on the area between -5000 to 5000 temp = df_initial[(df_initial["Quantity"]<5000) & (df_initial["Quantity"]>-5000)] plt.figure() plt.hist(temp["Quantity"], bins = 100) plt.yscale("log") plt.xlabel("Quantity") plt.ylabel("Count") plt.title("Distribution of Quantity") plt.show() # 5. InvoiceDate plt.figure() temp = pd.DataFrame(df_initial["InvoiceDate"].value_counts()).reset_index() plt.bar(temp["index"][:15].astype(str), temp.InvoiceDate[:15]) plt.xticks(rotation = 90) plt.xlabel("InvoiceDate") plt.ylabel("Count") plt.title("Count of Date") plt.show() # 6. UnitPrice plt.figure() plt.hist(df_initial["UnitPrice"], bins = 100) #plt.xticks(rotation = 90) #plt.xlim(-200,200) plt.yscale("log") plt.xlabel("Price") plt.ylabel("Count") plt.title("Distribution of UnitPrice") plt.show() # Zoom in on the area between 0 to 2000 temp = df_initial[(df_initial["UnitPrice"]<2000) & (df_initial["UnitPrice"]>0)] plt.figure() plt.hist(temp["UnitPrice"], bins = 100) plt.yscale("log") plt.xlabel("Price") plt.ylabel("Count") plt.title("Distribution of UnitPrice") plt.show() # 7. CustomerID plt.figure() temp = pd.DataFrame(df_initial["CustomerID"].value_counts()) plt.bar(temp.index[:15], temp.CustomerID[:15]) plt.xticks(rotation = 90) plt.xlabel("CustomerID") plt.ylabel("Count") plt.title("Count of CustomerID") plt.show() # 8. Country plt.figure() temp = pd.DataFrame(df_initial["Country"].value_counts()) plt.bar(temp.index[:15], temp.Country[:15]) plt.yscale("log") plt.xticks(rotation = 90) plt.xlabel("Country") plt.ylabel("Count") plt.title("Count of Country") plt.show() """ ### Look at Countries """ """ Here, I quickly look at the countries from which orders were made: """ temp = df_initial[['CustomerID', 'InvoiceNo', 'Country']].groupby(['CustomerID', 'InvoiceNo', 'Country']).count() temp = temp.reset_index(drop = False) countries = temp['Country'].value_counts() print('Number of countries in the data: {}'.format(len(countries))) """ and show the result on a chloropleth map: """ data = dict(type='choropleth', locations = countries.index, locationmode = 'country names', z = countries, text = countries.index, colorbar = {'title':'Order nb.'}, colorscale=[[0, 'rgb(224,255,255)'], [0.01, 'rgb(166,206,227)'], [0.02, 'rgb(31,120,180)'], [0.03, 'rgb(178,223,138)'], [0.05, 'rgb(51,160,44)'], [0.10, 'rgb(251,154,153)'], [0.20, 'rgb(255,255,0)'], [1, 'rgb(227,26,28)']], reversescale = False) #_______________________ layout = dict(title='Number of orders per country', geo = dict(showframe = True, projection={'type':'mercator'})) #______________ choromap = go.Figure(data = [data], layout = layout) iplot(choromap, validate=False) # I also explored which countries accounted for the most unique customers forhist = temp.groupby('Country')['CustomerID'].nunique().reset_index().sort_values('CustomerID', ascending = False) forhist.reset_index(inplace = True) forhist.drop('index', axis = 1, inplace = True) display(forhist.head(10)) """ We see that the dataset is largely dominated by orders made from the UK. ___ ### Look at Customers and products The dataframe contains $\sim$400,000 entries. What are the number of users and products in these entries ? """ # String to be searched in start of string search ="C" # boolean series returned df_initial[df_initial["InvoiceNo"].str.startswith(search)] pd.DataFrame([{'products': len(df_initial['StockCode'].value_counts()), 'transactions': len(df_initial['InvoiceNo'].value_counts()), 'customers': len(df_initial['CustomerID'].value_counts()), }], columns = ['products', 'transactions', 'customers'], index = ['quantity']) """ It can be seen that the data has 4372 users and that they bought 3684 different products. The total number of transactions carried out is of the order of $\sim$22'000. Now I will determine the number of products purchased in every transaction: """ temp = df_initial.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)['InvoiceDate'].count() nb_products_per_basket = temp.rename(columns = {'InvoiceDate':'Number of products'}) nb_products_per_basket[:10].sort_values('CustomerID') """ The first lines of this list shows several things worthy of interest: - the existence of entries with the prefix C for the **InvoiceNo** variable: this indicates transactions that have been canceled - the existence of users who only came once and only purchased one product (e.g. nº12346) - the existence of frequent users that buy a large number of items at each order """ """ # 2. Remove Cancelled Orders and Look at Remaining Records First of all, I count the number of transactions corresponding to canceled orders: """ nb_products_per_basket['order_canceled'] = nb_products_per_basket['InvoiceNo'].apply(lambda x:int('C' in x)) display(nb_products_per_basket[:5]) #______________________________________________________________________________________________ n1 = nb_products_per_basket['order_canceled'].sum() n2 = nb_products_per_basket.shape[0] print('Number of cancellation transactions among all orders: {}/{} ({:.2f}%) '.format(n1, n2, n1/n2*100)) """ We note that the number of cancellations is quite large ($\sim$16% of the total number of transactions). Now, let's look at the first lines of the dataframe: """ display(df_initial.sort_values('CustomerID')[:5]) """ On these few lines, we see that when an order is canceled, we have another transaction in the dataframe, mostly identical except for the **Quantity** and **InvoiceDate** variables. I decide to check if this is true for all the entries. To do this, I decide to locate the entries that indicate a negative quantity and check if there is *systematically* an order indicating the same quantity (but positive), with the same description (**CustomerID**, **Description** and **UnitPrice**): """ df_check = df_initial[df_initial['Quantity'] < 0][['CustomerID','Quantity', 'StockCode','Description','UnitPrice']] for index, col in df_check.iterrows(): if df_initial[(df_initial['CustomerID'] == col[0]) & (df_initial['Quantity'] == -col[1]) & (df_initial['Description'] == col[2])].shape[0] == 0: print(df_check.loc[index]) print(15*'-'+'>'+' HYPOTHESIS NOT FULFILLED') break """ We see that the initial hypothesis is not fulfilled because of the existence of a '_Discount_' entry. I check again the hypothesis but this time discarding the '_Discount_' entries: """ df_check = df_initial[(df_initial['Quantity'] < 0) & (df_initial['Description'] != 'Discount')][ ['CustomerID','Quantity','StockCode', 'Description','UnitPrice']] for index, col in df_check.iterrows(): if df_initial[(df_initial['CustomerID'] == col[0]) & (df_initial['Quantity'] == -col[1]) & (df_initial['Description'] == col[2])].shape[0] == 0: print(index, df_check.loc[index]) print(15*'-'+'>'+' HYPOTHESIS NOT FULFILLED') break """ Once more, we find that the initial hypothesis is not verified. Hence, cancellations do not necessarily correspond to orders that would have been made beforehand. At this point, I decide to create a new field in the dataframe that indicates if part of the order has been canceled. For the cancellations without counterparts, a few of them are probably due to the fact that the buy orders were performed before December 2010 (the point of entry of the database). Below, I make a census of the canceled orders and check for the existence of counterparts: """ # try to match all cancellation orders with all previous purchase orders cancels=df_initial[(df_initial['Description']!='Discount')&(df_initial['Quantity']<0)].copy() cancels['cancel_idx']=cancels.index df_match=df_initial.copy() df_match['match_idx']=df_match.index df_match=df_match[df_match['Quantity']>0] mg=cancels.merge(df_match[['InvoiceDate','CustomerID','StockCode','Quantity','match_idx']],how='left', left_on=['CustomerID','StockCode'],right_on=['CustomerID','StockCode']) matched=mg[mg['InvoiceDate_x']>mg['InvoiceDate_y']] matched['match_idx']=matched['match_idx'].astype(int) matched['Quantity_y']=matched['Quantity_y'].astype(int) # sort data to make sure matching starts with the latest purchase order matched=matched.sort_values(['cancel_idx','InvoiceDate_y'],ascending=[1,0]) # since one purchase order can be matched to multiple cancellation orders, # it can happen that one purchase order is accounted for too many cancellation orders # e.g. the purchase order only listed 8 units in quantity, # but is used to match multiple cancallation orders of which the quantities sum up to more than 8 # in other words, this purchase order is double counted # the following codes handle this problem count=1 while True: # if a customer returns items that were bought through multiple purchases # start accounts for the returned quantity from the latest order # until all returned/canceled quantity is accounted for and linked to a purchase order # it is possible that not all the canceled quantity can be accounted for matched['tot_bought']=matched.groupby('cancel_idx')['Quantity_y'].cumsum() matched['tot_bought_lag1']=matched.groupby('cancel_idx')['tot_bought'].shift(1).fillna(0) matched['QuantityCanceled']=matched[['Quantity_x','Quantity_y','tot_bought','tot_bought_lag1']].\ apply(lambda x: min(-x[0],x[1]) if x[3]==0 else \ (x[1] if x[2]<=-x[0] else \ (0 if x[3]>=-x[0] else -x[0]-x[3])), axis=1).astype(int) # add a column to track the total quantity a given purchae order has been used to account for a cancellation # if the total quantity counted exceeds the number of units that were bought in this order # remove this purchase order from the candidate lists of all cancellation orders from this point on # iterate one more time to link cancellations to purchases matched['tot_QuantityCanceled']=matched.groupby('match_idx')['QuantityCanceled'].cumsum() double_count=len(matched[matched['tot_QuantityCanceled']>matched['Quantity_y']]) print('Iteration {} double counted: {} times'.format(count, double_count)) if double_count>0: matched=matched[matched['tot_QuantityCanceled']<=matched['Quantity_y']] count+=1 else: print('End iteration') break tot_canceled=matched.groupby('cancel_idx')['Quantity_x'].mean()*(-1) tot_bought=matched.groupby('cancel_idx')['Quantity_y'].sum() # some but not all units of the cancellation are accounted for and linked to purchase orders partial_doubtful_entry=tot_canceled[tot_canceled>tot_bought].index.tolist() # all units of the cancellation are accounted for and linked to purchase orders entry_to_remove=tot_canceled[tot_canceled<=tot_bought].index.tolist() # no previous purchase that match the cancellation can be found doubtful_entry=list(set(mg['cancel_idx'].unique())-set(partial_doubtful_entry)-set(entry_to_remove)) df_cleaned = df_initial.copy(deep = True) df_cleaned=df_cleaned.merge(pd.DataFrame(matched[['match_idx','tot_QuantityCanceled']].groupby('match_idx').max()), how='left',left_index=True, right_index=True) df_cleaned.rename(columns={'tot_QuantityCanceled':'QuantityCanceled'},inplace=True) df_cleaned['QuantityCanceled']=df_cleaned['QuantityCanceled'].fillna(0).astype(int) df_cleaned.head() print(f"Entry_to_remove: {len(entry_to_remove)} ({len(entry_to_remove)/len(df_initial)*100:.2f}%)") print(f"Doubtful_entry: {len(doubtful_entry)} ({len(doubtful_entry)/len(df_initial)*100:.2f}%)") print(f"Partial_doubtful_entry: {len(partial_doubtful_entry)} ({len(partial_doubtful_entry)/len(df_initial)*100:.2f}%)") """ Among all the cancellation entries, the lines listed in the `entry_to_remove` list are the cancellation orders in which all the units canceled are fully accounted for. The lines listed in the `doubtful_entry` list correspond to the cancellations that no matching purchase order beforehand can be found. The lines listed in the ` partial_doubtful_entry` list represent cancellation orders in which the units canceled are partially accounted. In practice, I decide to delete all of these entries, which count respectively for $\sim$1.9%, 0.3% and 0.01% of the dataframe entries. Now all transactions that are not discount but have negative quantities have been removed. """ df_cleaned.drop(entry_to_remove, axis = 0, inplace = True) df_cleaned.drop(doubtful_entry, axis = 0, inplace = True) df_cleaned.drop(partial_doubtful_entry, axis = 0, inplace = True) remaining_entries = df_cleaned[(df_cleaned['Quantity'] < 0) & (df_cleaned['StockCode'] != 'D')] print("number of entries to delete: {}".format(remaining_entries.shape[0])) """ We see that the quantity canceled is greater than the sum of the previous purchases. """ # Output a profile of the cleaned data # profile = pandas_profiling.ProfileReport(df_cleaned) # profile.to_file('Cleaned data profile.html') df_cleaned['Month'] = df_cleaned["InvoiceDate"].map(lambda x: x.month) df_cleaned['Weekday'] = df_cleaned["InvoiceDate"].map(lambda x: x.weekday()) df_cleaned['Day'] = df_cleaned["InvoiceDate"].map(lambda x: x.day) df_cleaned['Hour'] = df_cleaned["InvoiceDate"].map(lambda x: x.hour) df_cleaned['Month'].value_counts() df_cleaned['Weekday'].value_counts() df_cleaned['Day'].value_counts() df_cleaned['Hour'].value_counts() plt.hist(df_cleaned['Month'], bins=12) plt.hist(df_cleaned['Weekday'], bins=7) plt.hist(df_cleaned['Day'], bins=31) plt.hist(df_cleaned['Hour'], bins=24) """ #### Look at StockCode Above, it has been seen that some values of the ** StockCode ** field indicate a particular transaction (i.e. D for _Discount_). I check the contents of this field by looking for the set of codes that would contain only letters: """ list_special_codes = df_cleaned[df_cleaned['StockCode'].str.contains('^[a-zA-Z]+', regex=True)]['StockCode'].unique() list_special_codes for code in list_special_codes: print("{:<15} -> {:<30}".format(code, df_cleaned[df_cleaned['StockCode'] == code]['Description'].unique()[0])) print("{} entries removed".format(len(df_cleaned[df_cleaned['StockCode'].isin(list_special_codes)]))) df_cleaned=df_cleaned[-df_cleaned['StockCode'].isin(list_special_codes)] """ We see that there are several types of peculiar transactions, connected e.g. to port charges or bank charges. For better product segmentation, I will remove such transactions. ___ #### Basket Price """ """ I create a new field that indicates the total price of every purchase: """ df_cleaned['TotalPrice'] = df_cleaned['UnitPrice'] * (df_cleaned['Quantity'] - df_cleaned['QuantityCanceled']) df_cleaned.sort_values('CustomerID')[:5] revenue_per_countries = df_cleaned.groupby(["Country"])["TotalPrice"].sum().sort_values() revenue_per_countries.plot(kind='bar', figsize=(15,12)) plt.yscale('log') plt.title("Revenue per Country") """ Each entry of the dataframe indicates prices for a single kind of product. Hence, orders are split on several lines. I collect all the purchases made during a single order to recover the total order price: """ #___________________________________________ # sum of purchases / user and orders temp = df_cleaned.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)['TotalPrice'].sum() basket_price = temp.rename(columns = {'TotalPrice':'Basket Price'}) #_____________________ # order date df_cleaned['InvoiceDate_int'] = df_cleaned['InvoiceDate'].astype('int64') temp = df_cleaned.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)['InvoiceDate_int'].mean() df_cleaned.drop('InvoiceDate_int', axis = 1, inplace = True) basket_price.loc[:, 'InvoiceDate'] = pd.to_datetime(temp['InvoiceDate_int']) #______________________________________ # selection of significant entries: # basket_price = basket_price[basket_price['Basket Price'] > 0] basket_price.sort_values('CustomerID')[:6] # Show distribution of basket prices in pie chart. # Purchase count price_range = [0, 50, 100, 200, 500, 1000, 5000, 50000] count_price = [] for i, price in enumerate(price_range): if i == 0: continue val = basket_price[(basket_price['Basket Price'] < price) & (basket_price['Basket Price'] > price_range[i-1])]['Basket Price'].count() count_price.append(val) # Representation of the number of purchases / amount plt.rc('font', weight='bold') f, ax = plt.subplots(figsize=(11, 6)) colors = ['yellowgreen', 'gold', 'wheat', 'c', 'violet', 'royalblue','firebrick'] labels = [ '{}<.<{}'.format(price_range[i-1], s) for i,s in enumerate(price_range) if i != 0] sizes = count_price explode = [0.0 if sizes[i] < 100 else 0.0 for i in range(len(sizes))] ax.pie(sizes, explode = explode, labels=labels, colors = colors, autopct = lambda x:'{:1.0f}%'.format(x) if x > 1 else '', shadow = False, startangle=0) ax.axis('equal') f.text(0.5, 1.01, "Distribution of order amounts", ha='center', fontsize = 18); # make histogram distributions of the basket prices plt.hist(basket_price['Basket Price'], bins=20) plt.yscale('log') xmax = 1000 # plt.yscale('log') temp = basket_price[basket_price['Basket Price'] <= xmax] plt.hist(temp['Basket Price'], bins=50) plt.show() """ It can be seen that the vast majority of orders concern relatively large purchases given that $\sim$65% of purchases give prices in excess of £ 200. There's interesting aspects to this distribution, for example a spike arounf 300, and a rise around 100. """ df_cleaned.Quantity.describe() """ # 3. Make Product Categories In the dataframe, products are uniquely identified through the **StockCode** field. A short description of the products is given in the **Description** field. In this section, I intend to use the content of this latter variable in order to group the products into different categories. ___ ### Look at Products Description As a first step, I extract from the **Description** field the information that will prove useful. To do this, I use the following function: """ is_noun = lambda pos: pos[:2] == 'NN' def keywords_inventory(dataframe, colonne = 'Description'): stemmer = nltk.stem.SnowballStemmer("english") keywords_roots = dict() # collect the words / root keywords_select = dict() # association: root <-> keyword category_keys = [] count_keywords = dict() icount = 0 for s in dataframe[colonne]: if pd.isnull(s): continue lines = s.lower() tokenized = nltk.word_tokenize(lines) nouns = [word for (word, pos) in nltk.pos_tag(tokenized) if is_noun(pos)] for t in nouns: t = t.lower() ; racine = stemmer.stem(t) if racine in keywords_roots: keywords_roots[racine].add(t) count_keywords[racine] += 1 else: keywords_roots[racine] = {t} count_keywords[racine] = 1 for s in keywords_roots.keys(): if len(keywords_roots[s]) > 1: min_length = 1000 for k in keywords_roots[s]: if len(k) < min_length: clef = k ; min_length = len(k) category_keys.append(clef) keywords_select[s] = clef else: category_keys.append(list(keywords_roots[s])[0]) keywords_select[s] = list(keywords_roots[s])[0] print("Nb of keywords in variable '{}': {}".format(colonne,len(category_keys))) return category_keys, keywords_roots, keywords_select, count_keywords """ This function takes as input the dataframe and analyzes the content of the **Description** column by performing the following operations: - extract the names (proper, common) appearing in the products description - for each name, I extract the root of the word and aggregate the set of names associated with this particular root - count the number of times each root appears in the dataframe - when several words are listed for the same root, I consider that the keyword associated with this root is the shortest name (this systematically selects the singular when there are singular/plural variants) The first step of the analysis is to retrieve the list of products: """ df_produits = pd.DataFrame(df_initial['Description'].unique()).rename(columns = {0:'Description'}) """ Once this list is created, I use the function I previously defined in order to analyze the description of the various products: """ keywords, keywords_roots, keywords_select, count_keywords = keywords_inventory(df_produits) """ The execution of this function returns three values: - `keywords`: the list of extracted keywords - `keywords_roots`: a dictionary where the keys are the keywords roots and the values are the lists of words associated with those roots - `count_keywords`: dictionary listing the number of times every word is used At this point, I convert the `count_keywords` dictionary into a list, to sort the keywords according to their occurences: """ list_products = [] for k,v in count_keywords.items(): list_products.append([keywords_select[k],v]) list_products.sort(key = lambda x:x[1], reverse = True) """ Using it, I create a representation of the most common keywords: """ liste = sorted(list_products, key = lambda x:x[1], reverse = True) #_______________________________ plt.rc('font', weight='normal') fig, ax = plt.subplots(figsize=(7, 25)) y_axis = [i[1] for i in liste[:125]] x_axis = [k for k,i in enumerate(liste[:125])] x_label = [i[0] for i in liste[:125]] plt.xticks(fontsize = 15) plt.yticks(fontsize = 13) plt.yticks(x_axis, x_label) plt.xlabel("Nb. of occurences", fontsize = 18, labelpad = 10) ax.barh(x_axis, y_axis, align = 'center',color='red') ax = plt.gca() ax.invert_yaxis() #_______________________________________________________________________________________ plt.title("Words occurence",bbox={'facecolor':'k', 'pad':5}, color='orange',fontsize = 25) plt.show() """ ___ ### Defining product categories """ """ The list that was obtained contains more than 1400 keywords and the most frequent ones appear in more than 200 products. However, while examinating the content of the list, I note that some names are useless. Others are do not carry information, like colors. Therefore, I discard these words from the analysis that follows and also, I decide to consider only the words that appear more than 13 times. """ # use predefined list of stopwords from NLTK package nltk_stopwords = set(stopwords.words('english')) # basic function that removes matched stopwords from a dictionary object def remove_stopwords(corpus: dict): temp_dict = {} # create a temporary dictionary tokens = keywords_select.keys() filtered_tokens = list(filter(lambda token: token not in nltk_stopwords, tokens)) # filter for stopwords for i in filtered_tokens: # update the temporary dictionary temp_dict.update({i: i}) print(f'{len(keywords_select.keys())-len(temp_dict.keys())} stopwords removed') return temp_dict keywords_select = remove_stopwords(keywords_select) # also prints how many stopwords were removed # reflect the change on 'count_keywords' dict, too count_keywords = {k: v for k, v in count_keywords.items() if k in keywords_select.keys()} # remove some very common words list_products = [] for k,v in count_keywords.items(): word = keywords_select[k] if word in ['pink', 'blue', 'tag', 'green', 'orange','candl','design','vintag',\ 'box', 'sign', 'vintage', 'glass', 'light', 'candle', 'flower']: continue if len(word) < 5 or v < 13: continue if ('+' in word) or ('/' in word): continue list_products.append([word, v]) #______________________________________________________ list_products.sort(key = lambda x:x[1], reverse = True) print('preserved words:', len(list_products)) liste = sorted(list_products, key = lambda x:x[1], reverse = True) #_______________________________ plt.rc('font', weight='normal') fig, ax = plt.subplots(figsize=(7, 25)) y_axis = [i[1] for i in liste[:125]] x_axis = [k for k,i in enumerate(liste[:125])] x_label = [i[0] for i in liste[:125]] plt.xticks(fontsize = 15) plt.yticks(fontsize = 13) plt.yticks(x_axis, x_label) plt.xlabel("Nb. of occurences", fontsize = 18, labelpad = 10) ax.barh(x_axis, y_axis, align = 'center',color='red') ax = plt.gca() ax.invert_yaxis() #_______________________________________________________________________________________ plt.title("Words occurence",bbox={'facecolor':'k', 'pad':5}, color='orange',fontsize = 25) plt.show() """ ____ #### Data encoding Now I will use these keywords to create groups of product. Firstly, I define the $X$ matrix as: """ """ | | mot 1 | ... | mot j | ... | mot N | |:-:|---|---|---|---|---| | product 1 | $a_{1,1}$ | | | | $a_{1,N}$ | | ... | | | ... | | | |product i | ... | | $a_{i,j}$ | | ... | |... | | | ... | | | | product M | $a_{M,1}$ | | | | $a_{M,N}$ | """ """ where the $a_ {i, j}$ coefficient is 1 if the description of the product $i$ contains the word $j$, and 0 otherwise. Here I have a row for each product and a column for each keyword. I fill in the entries with zero or 1, depending on whether that product has that keyword in its description. """ liste_produits = df_cleaned['Description'].unique() X = pd.DataFrame() for key, occurence in list_products: X.loc[:, key] = list(map(lambda x:int(key.upper() in x), liste_produits)) X """ The $X$ matrix indicates the words contained in the description of the products using the *one-hot-encoding* principle. In practice, I have found that introducing the price range results in more balanced groups in terms of element numbers. Hence, I add 6 extra columns to this matrix, where I indicate the price range of the products: """ threshold = [0, 1, 2, 3, 5, 10] label_col = [] for i in range(len(threshold)): if i == len(threshold)-1: col = '.>{}'.format(threshold[i]) else: col = '{}<.<{}'.format(threshold[i],threshold[i+1]) label_col.append(col) X.loc[:, col] = 0 def categorize_price(price): j=0 while price > threshold[j+1]: j+=1 if j+1 == len(threshold): break return label_col[j] prod_prices=df_cleaned[['Description','UnitPrice']].groupby('Description').mean() prod_prices['PriceRange']=prod_prices['UnitPrice'].apply(categorize_price) for col in label_col: prods=prod_prices[prod_prices['PriceRange']==col].index X.loc[[list(liste_produits).index(prod) for prod in prods],col]=1 X """ and to choose the appropriate ranges, I check the number of products in the different groups: """ # Return frequency of products by price range. X.iloc[:,-6:].sum() # print("{:<8} {:<20} \n".format('gamme', 'nb. produits') + 20*'-') # for i in range(len(threshold)): # if i == len(threshold)-1: # col = '.>{}'.format(threshold[i]) # else: # col = '{}<.<{}'.format(threshold[i],threshold[i+1]) # print("{:<10} {:<20}".format(col, X.loc[:, col].sum())) prod_prices.groupby('PriceRange').size() """ ____ #### Create clusters of products In this section, I will group the products into different classes. In the case of matrices with binary encoding, cosine similarity/distance is a commonly-used measure of distance. Since **kmeans** method of sklearn uses a Euclidean distance, I defined my own functions to perform kmeans clustering using cosine distance. In order to define (approximately) the number of clusters that best represents the data, I use the silhouette score as a measure of the "quality" of the segmentation. """ def calculate_cosine_similarity(a,b): """Calculates the cosine similarity between two vectors""" dot_product=np.dot(a,b) len_a=np.sqrt(np.dot(a,a)) len_b=np.sqrt(np.dot(b,b)) return dot_product/(len_a*len_b+1e-9) def get_closest_center(v,centers): """Find the center that given vector is closest to based on cosine similarity""" distances=[calculate_cosine_similarity(c,v) for c in centers] return distances.index(max(distances)) def get_center(c): """Calculate the center of a list of vectors by averaging the vectors after normalizing them to unit length""" return np.array([[i/np.sqrt(sum(i*i))] for i in c]).mean(axis=0)[0] class KMeans_cosine(object): """KMeans clustering based on cosine distance""" def __init__(self,vectors,k,max_iter=30,min_sample=10,init_centers=None): self.vectors=vectors self.n=len(self.vectors) self.max_iter=max_iter self.min_sample=min_sample if init_centers is None: self.k=k self.centers=self.vectors[np.random.choice(self.n,self.k)] else: self.centers=init_centers self.k=len(init_centers) self.clusters=[[] for c in range(self.k)] self.labels=[] def update_cluster(self): """Assign vectors to the centers they are closest to""" labels=[] for v in self.vectors: index=get_closest_center(v,self.centers) labels.append(index) self.clusters[index].append(v) self.labels=labels def update_center(self): """Calculate the center of a list of vectors""" if min([len(c) for c in self.clusters])<self.min_sample: self.centers=self.vectors[np.random.choice(self.n,self.k)] self.clusters=[[] for c in range(self.k)] self.labels=[] return True new_centers=[] for c in self.clusters: center=get_center(c) new_centers.append(center) new_centers=np.array(new_centers) if (self.centers==new_centers).all(): return False self.centers=new_centers self.clusters=[[] for c in range(self.k)] return True def fit(self): """Iteratively move centers until they have stabilized or the maximum number of iteration has reached""" self.update_cluster() count=0 while self.update_center(): self.update_cluster() count+=1 if count==self.max_iter: break """ The cosine kmeans method I constructed is somewhat unstable. Thus I performed 100 iterations for n_clusters from 3 to 8 to get a more stablized picture. The results are: - For n_clusters = 3 The average silhouette_score is : 0.25673884714979833 - For n_clusters = 4 The average silhouette_score is : 0.32045152822154677 - For n_clusters = 5 The average silhouette_score is : 0.3406799276763692 - For n_clusters = 6 The average silhouette_score is : 0.38970683534465045 - For n_clusters = 7 The average silhouette_score is : 0.3687191429652772 - For n_clusters = 8 The average silhouette_score is : 0.35286912641983464 I chose 6 for it has the highest average silhouette score across all iterations. After that, I ran a few more iterations to find an near-optimal model with an average silhouette score above a particular threshold. """ # generate the cosine distance matrix dist_matrix=cdist(X,X,'cosine') # iter_n=100 # avg_silhouette_scores={} # for n_clusters in range(3,9): # tot_score=0 # for i in range(iter_n): # kmeans = KMeans_cosine(X.values,n_clusters,min_sample=50) # kmeans.fit() # clusters=kmeans.labels # silhouette_avg = silhouette_score(dist_matrix, clusters, metric='precomputed') # tot_score+=silhouette_avg # avg_score=tot_score/iter_n # avg_silhouette_scores[n_clusters]=avg_score # print("For n_clusters =", n_clusters, "The average silhouette_score is :", avg_score) n_clusters=6 silhouette_avg=0 # init_centers=pd.read_csv('init_centers.csv').values niter = 0 while silhouette_avg<0.53: #init_centers is provided to replicate one optimal solution I found through many iterations # kmeans = KMeans_cosine(X.values,n_clusters,min_sample=50,init_centers=init_centers) kmeans = KMeans_cosine(X.values,n_clusters,min_sample=50) kmeans.fit() clusters=kmeans.labels silhouette_avg = silhouette_score(dist_matrix, clusters, metric='precomputed') print(niter, "The average silhouette_score is :", silhouette_avg) niter = niter+1 clusters=np.array(clusters) """ #### Examine the clusters """ """ I checked the number of elements in each class and did a sanity check by simply looking at the products in a cluster. """ prods=pd.DataFrame([liste_produits,clusters]).T prods.columns=['prod','cluster'] prods['cluster'].value_counts() """ It's worth noting that because the majority of the matrix is sparse and a product must be categorized to one of the price baskets, the clustering result is dominated by the price range. In one extreme iteration, the six clusters are exactly separated by the price range. """ prods[prods['cluster']==0].head(20) prods[prods['cluster']==1].head(20) prods[prods['cluster']==2].head(20) prods[prods['cluster']==3].head(20) prods[prods['cluster']==4].head(20) prods[prods['cluster']==5].head(20) cluster_price=X[['0<.<1', '1<.<2', '2<.<3', '3<.<5','5<.<10', '.>10']].copy() cluster_price['cluster']=clusters cluster_price.groupby('cluster').mean() """ ** a / _Silhouette intra-cluster score_ ** In order to have an insight on the quality of the classification, we can represent the silhouette scores of each element of the different clusters. This is the purpose of the next figure which is taken from the [sklearn documentation](http://scikit-learn.org/stable/auto_examples/cluster/plot_kmeans_silhouette_analysis.html): """ def graph_component_silhouette(n_clusters, lim_x, mat_size, sample_silhouette_values, clusters): # plt.rcParams["patch.force_edgecolor"] = True plt.style.use('fivethirtyeight') mpl.rc('patch', edgecolor = 'dimgray', linewidth=1) #____________________________ fig, ax1 = plt.subplots(1, 1) fig.set_size_inches(8, 8) ax1.set_xlim([lim_x[0], lim_x[1]]) ax1.set_ylim([0, mat_size + (n_clusters + 1) * 10]) y_lower = 10 for i in range(n_clusters): #___________________________________________________________________________________ # Aggregate the silhouette scores for samples belonging to cluster i, and sort them ith_cluster_silhouette_values = sample_silhouette_values[clusters == i] ith_cluster_silhouette_values.sort() size_cluster_i = ith_cluster_silhouette_values.shape[0] y_upper = y_lower + size_cluster_i cmap = cm.get_cmap("Spectral") color = cmap(float(i) / n_clusters) ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values, facecolor=color, edgecolor=color, alpha=0.8) #____________________________________________________________________ # Label the silhouette plots with their cluster numbers at the middle ax1.text(-0.03, y_lower + 0.5 * size_cluster_i, str(i), color = 'red', fontweight = 'bold', bbox=dict(facecolor='white', edgecolor='black', boxstyle='round, pad=0.3')) #______________________________________ # Compute the new y_lower for next plot y_lower = y_upper + 10 #____________________________________ # define individual silouhette scores sample_silhouette_values = silhouette_samples(dist_matrix, clusters,metric="precomputed") #__________________ # and do the graph graph_component_silhouette(n_clusters, [-0.05, 0.7], len(X), sample_silhouette_values, clusters) """ ** b/ _Word Cloud_** Now we can have a look at the type of objects that each cluster represents. In order to obtain a global view of their contents, I determine which keywords are the most frequent in each of them """ liste = pd.DataFrame(liste_produits) liste_words = [word for (word, occurence) in list_products] occurence = [dict() for _ in range(n_clusters)] for i in range(n_clusters): liste_cluster = liste.loc[clusters == i] for word in liste_words: if word in ['art', 'set', 'heart', 'pink', 'blue', 'tag',]: continue occurence[i][word] = sum(liste_cluster.loc[:, 0].str.contains(word.upper())) """ and I output the result as wordclouds: """ #________________________________________________________________________ def random_color_func(word=None, font_size=None, position=None, orientation=None, font_path=None, random_state=None): h = int(360.0 * tone / 255.0) s = int(100.0 * 255.0 / 255.0) l = int(100.0 * float(random_state.randint(70, 120)) / 255.0) return "hsl({}, {}%, {}%)".format(h, s, l) #________________________________________________________________________ def make_wordcloud(liste, increment): ax1 = fig.add_subplot(4,2,increment) words = dict() trunc_occurences = liste[0:150] for s in trunc_occurences: words[s[0]] = s[1] #________________________________________________________ wordcloud = WordCloud(width=1000,height=400, background_color='lightgrey', max_words=1628,relative_scaling=1, color_func = random_color_func, normalize_plurals=False) wordcloud.generate_from_frequencies(words) ax1.imshow(wordcloud, interpolation="bilinear") ax1.axis('off') plt.title('cluster nº{}'.format(increment-1)) #________________________________________________________________________ fig = plt.figure(1, figsize=(14,14)) color = [0, 160, 130, 95, 280, 40, 330, 110, 25] for i in range(n_clusters): list_cluster_occurences = occurence[i] tone = color[i] # define the color of the words liste = [] for key, value in list_cluster_occurences.items(): liste.append([key, value]) liste.sort(key = lambda x:x[1], reverse = True) make_wordcloud(liste, i+1) """ Even with fine tuning and many iterations, we can still find many words appearing in various clusters. This imperfect result is expected because at the end of the day machines can only do what we tell them to do, in our case here, that is to find similar words based on how they are spelled, not what they mean. The machine has no way to know "cushion" is closer to "cabinet" than it is to "bracelet". In real world, it's very unlikely a business will do unsupervised product segmentation like this. ** c / _Principal Component Analysis_ ** In order to ensure that these clusters are reasonably distinct, I look at their composition in a relevant low dimensional space. Given the large number of variables of the initial matrix, I first perform a PCA: """ # Perform PCA to subsequently check distinctness between clusters. matrix=X.values pca = PCA() pca.fit(matrix) pca_samples = pca.transform(matrix) """ and then check for the amount of variance explained by each component: """ fig, ax = plt.subplots(figsize=(14, 5)) sns.set(font_scale=1) plt.step(range(matrix.shape[1]), pca.explained_variance_ratio_.cumsum(), where='mid', label='cumulative explained variance') sns.barplot(np.arange(1,matrix.shape[1]+1), pca.explained_variance_ratio_, alpha=0.5, color = 'green', label='individual explained variance') plt.xlim(0, 100) ax.set_xticklabels([s if int(s.get_text())%2 == 0 else '' for s in ax.get_xticklabels()]) plt.ylabel('Explained variance', fontsize = 14) plt.xlabel('Principal components', fontsize = 14) plt.legend(loc='upper left', fontsize = 13); """ We see that the number of components required to explain the data is extremely important: we need more than 100 components to explain 90% of the variance of the data. So we find that the space contains lots of variance in many non correlated directions, and PCA doesn't really help that much. In practice, I decide to keep only a limited number of components since this decomposition is only performed to visualize the data: """ pca = PCA(n_components=50) matrix_9D = pca.fit_transform(matrix) mat = pd.DataFrame(matrix_9D) mat['cluster'] = pd.Series(clusters) import matplotlib.patches as mpatches sns.set_style("white") sns.set_context("notebook", font_scale=1, rc={"lines.linewidth": 2.5}) LABEL_COLOR_MAP = {0:'r', 1:'gold', 2:'b', 3:'k', 4:'c', 5:'g'} label_color = [LABEL_COLOR_MAP[l] for l in mat['cluster']] fig = plt.figure(figsize = (15,8)) increment = 0 for ix in range(4): for iy in range(ix+1, 4): increment += 1 ax = fig.add_subplot(2,3,increment) ax.scatter(mat[ix], mat[iy], c= label_color, alpha=0.4) plt.ylabel('PCA {}'.format(iy+1), fontsize = 12) plt.xlabel('PCA {}'.format(ix+1), fontsize = 12) ax.yaxis.grid(color='lightgray', linestyle=':') ax.xaxis.grid(color='lightgray', linestyle=':') ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) if increment == 9: break if increment == 9: break #_______________________________________________ # I set the legend: abreviation -> airline name comp_handler = [] for i in range(5): comp_handler.append(mpatches.Patch(color = LABEL_COLOR_MAP[i], label = i)) plt.legend(handles=comp_handler, bbox_to_anchor=(1.1, 0.97), title='Cluster', shadow = True, frameon = True, framealpha = 1, fontsize = 13, bbox_transform = plt.gcf().transFigure) plt.show() """ ___ # 4. Make Customer Categories ### Formatting data In the previous section, the different products were grouped in five clusters. In order to prepare the rest of the analysis, a first step consists in introducing this information into the dataframe. To do this, I create the categorical variable **categ_product** where I indicate the cluster of each product : """ # Append product category per above clustering to dataframe. corresp = dict() for key, val in zip (liste_produits, clusters): corresp[key] = val #__________________________________________________________________________ df_cleaned['categ_product'] = df_cleaned.loc[:, 'Description'].map(corresp) df_cleaned """ ___ #### Grouping products In a second step, I decide to create the **categ_N** variables (with $ N \in [0: 5]$) that contains the amount spent in each product category: """ for i in range(6): col = 'categ_{}'.format(i) df_temp = df_cleaned[df_cleaned['categ_product'] == i] price_temp = df_temp['UnitPrice'] * (df_temp['Quantity'] - df_temp['QuantityCanceled']) price_temp = price_temp.apply(lambda x:x if x > 0 else 0) df_cleaned.loc[:, col] = price_temp df_cleaned[col].fillna(0, inplace = True) #__________________________________________________________________________________________________ df_cleaned[['InvoiceNo', 'Description', 'categ_product', 'categ_0', 'categ_1', 'categ_2', 'categ_3','categ_4','categ_5']][:5] """ By sales revenue, category 0, 1 and 3 are the top three categories. """ df_cleaned[['categ_0', 'categ_1', 'categ_2', 'categ_3','categ_4','categ_5']].sum() """ Up to now, the information related to a single order was split over several lines of the dataframe (one line per product). I decide to collect the information related to a particular order and put in in a single entry. I therefore create a new dataframe that contains, for each order, the amount of the basket, as well as the way it is distributed over the 5 categories of products: """ # Create dataframe which combines purchase information for each distinct Invoice Number. #___________________________________________ # sum of purchases / user & order temp = df_cleaned.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)['TotalPrice'].sum() basket_price = temp.rename(columns = {'TotalPrice':'Basket Price'}) #____________________________________________________________ # percentage of the price of the order / product category for i in range(n_clusters): col = 'categ_{}'.format(i) temp = df_cleaned.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)[col].sum() basket_price.loc[:, col] = temp.loc[:, col] #_____________________ # order date df_cleaned['InvoiceDate_int'] = df_cleaned['InvoiceDate'].astype('int64') temp = df_cleaned.groupby(by=['CustomerID', 'InvoiceNo'], as_index=False)['InvoiceDate_int'].mean() df_cleaned.drop('InvoiceDate_int', axis = 1, inplace = True) basket_price.loc[:, 'InvoiceDate'] = pd.to_datetime(temp['InvoiceDate_int']) #______________________________________ # selection of significant entries: basket_price = basket_price[basket_price['Basket Price'] > 0] basket_price.sort_values('CustomerID', ascending = True)[:5] """ #### Separation of data over time The dataframe `basket_price` contains information for a period of 12 months. Later, one of the objectives will be to develop a model capable of characterizing and anticipating the habits of the customers visiting the site and this, from their first visit. In order to be able to test the model in a realistic way, I split the data set by retaining the first 10 months to develop the model and the following two months to test it: """ print(basket_price['InvoiceDate'].min(), '->', basket_price['InvoiceDate'].max()) temp1 = datetime(2011,10,1) temp = np.datetime64(datetime(2011,10,1)) print(temp) type(temp) print(basket_price['InvoiceDate']) set_entrainement = basket_price[basket_price['InvoiceDate'] < np.datetime64(datetime(2011,10,1))] set_test = basket_price[basket_price['InvoiceDate'] >= np.datetime64(datetime(2011,10,1))] basket_price.head() """ ____ #### Consumer Order Combinations In a second step, I group together the different entries that correspond to the same user. I thus determine the number of purchases made by the user, as well as the minimum, maximum, average amounts and the total amount spent during all the visits. For each customer, I also added the total dollar value of all his or her cancelled items as a percentage of total purchase amount, as well as the number of distinct products bought. """ # Group all relevent information together for each customer. #________________________________________________________________ # number of visits and stats on the cart amount / users transactions_per_user=basket_price.groupby(by=['CustomerID'])['Basket Price'].agg(['count','min','max','mean','sum']) for i in range(n_clusters): col = 'categ_{}'.format(i) transactions_per_user.loc[:,col] = basket_price.groupby(by=['CustomerID'])[col].sum() /\ transactions_per_user['sum']*100 df_cleaned['CanceledValue']=df_cleaned['QuantityCanceled']*df_cleaned['UnitPrice'] transactions_per_user=transactions_per_user.merge( df_cleaned[['CustomerID','CanceledValue']].groupby('CustomerID').sum(), left_index=True,right_index=True) transactions_per_user['canceled%']=transactions_per_user['CanceledValue']/transactions_per_user['sum']*100 transactions_per_user['canceled%']=transactions_per_user['canceled%'].apply(lambda x: 100 if x>100 else x) transactions_per_user=transactions_per_user.merge( df_cleaned[['Description','CustomerID']].groupby('CustomerID').nunique().\ rename(columns={'Description':'unique_prod'}), left_index=True,right_index=True) transactions_per_user.drop(columns='CanceledValue',inplace=True) transactions_per_user.index.name = None transactions_per_user.reset_index(drop = False, inplace = True) # transactions_per_user.drop(columns='CustomerID', inplace=True) transactions_per_user.rename(columns={'index':'CustomerID'}, inplace=True) transactions_per_user.sort_values('CustomerID', ascending = True)[:5] """ Finally, I define two additional variables that give the number of days elapsed since the first purchase (** FirstPurchase **) and the number of days since the last purchase (** LastPurchase **): """ last_date = basket_price['InvoiceDate'].max().date() first_registration = pd.DataFrame(basket_price.groupby(by=['CustomerID'])['InvoiceDate'].min()) last_purchase = pd.DataFrame(basket_price.groupby(by=['CustomerID'])['InvoiceDate'].max()) test = first_registration.applymap(lambda x:(last_date - x.date()).days) test2 = last_purchase.applymap(lambda x:(last_date - x.date()).days) transactions_per_user.loc[:, 'LastPurchase'] = test2.reset_index(drop = False)['InvoiceDate'] transactions_per_user.loc[:, 'FirstPurchase'] = test.reset_index(drop = False)['InvoiceDate'] transactions_per_user[:5] """ 95% customers returned less than 10% of their purchase in value, purchased less than 205 different products and visited the shop less than 13 times. """ print('"Canceled %" 95 quantile:',transactions_per_user['canceled%'].quantile(0.95)) transactions_per_user['canceled%'].hist() plt.yscale('log') print('"Unique Products" 95 quantile:',transactions_per_user['unique_prod'].quantile(0.95)) transactions_per_user['unique_prod'].hist(bins=50) plt.yscale('log') print('"Count" 95 quantile:',transactions_per_user['count'].quantile(0.95)) transactions_per_user['count'].hist(bins=20) plt.yscale('log') print('"Mean" 95 quantile:',transactions_per_user['mean'].quantile(0.95)) transactions_per_user['mean'].hist(bins=50) plt.yscale('log') transactions_per_user.describe() transactions_per_user.quantile(0.97) """ I capped the long tails at around 97% quantile. """ transactions_per_user['count']=transactions_per_user['count'].clip(upper=20) transactions_per_user['min']=transactions_per_user['min'].clip(upper=1000) transactions_per_user['max']=transactions_per_user['max'].clip(upper=2000) transactions_per_user['mean']=transactions_per_user['mean'].clip(upper=1500) transactions_per_user['canceled%']=transactions_per_user['canceled%'].clip(upper=15) transactions_per_user['unique_prod']=transactions_per_user['unique_prod'].clip(upper=300) transactions_per_user.plot.scatter('min','max',color='yellow') transactions_per_user.plot.scatter('LastPurchase','FirstPurchase',color='green') temp = transactions_per_user[transactions_per_user['FirstPurchase'] > 200] fig = plt.hist(temp['FirstPurchase'], bins = 100,color='violet') transactions_per_user.plot.scatter('categ_0','categ_1',color='red') """ A customer category of particular interest is that of customers who make only one purchase. I find that this type of customer represents 1/3 of the customers listed. When doing customer segmentation, I'll put all these "one-time-buyers" into their own segment. """ n1 = transactions_per_user[transactions_per_user['count'] == 1].shape[0] n2 = transactions_per_user.shape[0] print("number of customers with a single purchase: {:<2}/{:<5} ({:<2.2f}%)".format(n1,n2,n1/n2*100)) print(f"number of customers with more than one purchases: {n2-n1}") """ Compared to multi-time buyers, one-time buyers have a lower canceled value percentage and slighly smaller average order size. In terms of product mix, there doesn't seem to be a big diffference. """ selected_customers = transactions_per_user[transactions_per_user['count'] > 1].copy(deep = True) single_buyers=transactions_per_user[transactions_per_user['count'] == 1] customer_comparison=pd.DataFrame([single_buyers.mean().values,selected_customers.mean().values], columns=single_buyers.columns, index=['one_time_buyers','multi_time_buyers']).drop(columns='CustomerID') customer_comparison """ ___ ### Creation of customers categories """ """ #### Data encoding The dataframe `transactions_per_user` contains a summary of all the commands that were made. Each entry in this dataframe corresponds to a particular client. I use this information to characterize the different types of customers and only keep a subset of variables: """ list_cols = ['count','min','max','mean','categ_0','categ_1','categ_2','categ_3','categ_4', 'categ_5', 'canceled%','unique_prod'] #_____________________________________________________________ matrix = selected_customers[list_cols].to_numpy() """ In practice, the different variables I selected have quite different ranges of variation and before continuing the analysis, I create a matrix where these data are standardized: """ scaler = StandardScaler() scaler.fit(matrix) print('variables mean values: \n' + 90*'-' + '\n' , scaler.mean_) scaled_matrix = scaler.transform(matrix) """ In the following, I will create clusters of customers. In practice, before creating these clusters, it is interesting to define a base of smaller dimension allowing to describe the `scaled_matrix` matrix. In this case, I will use this base in order to create a representation of the different clusters and thus verify the quality of the separation of the different groups. I therefore perform a PCA beforehand: """ pca = PCA() pca.fit(scaled_matrix) pca_samples = pca.transform(scaled_matrix) """ and I represent the amount of variance explained by each of the components: """ fig, ax = plt.subplots(figsize=(14, 5)) sns.set(font_scale=1) plt.step(range(matrix.shape[1]), pca.explained_variance_ratio_.cumsum(), where='mid', label='cumulative explained variance') sns.barplot(np.arange(1,matrix.shape[1]+1), pca.explained_variance_ratio_, alpha=0.5, color = 'g', label='individual explained variance') plt.xlim(0, 10) ax.set_xticklabels([s if int(s.get_text())%2 == 0 else '' for s in ax.get_xticklabels()]) plt.ylabel('Explained variance', fontsize = 14) plt.xlabel('Principal components', fontsize = 14) plt.legend(loc='best', fontsize = 13); """ ___ #### Creation of customer categories """ """ At this point, I define clusters of clients from the standardized matrix that was defined earlier and using the `k-means` algorithm from` scikit-learn`. I choose the number of clusters based on the silhouette score and I find that a pretty good score is obtained with 8 clusters: """ for n in range(6,12): kmeans = KMeans(init='k-means++', n_clusters = n, n_init=100) kmeans.fit(scaled_matrix) clusters_clients = kmeans.predict(scaled_matrix) silhouette_avg = silhouette_score(scaled_matrix, clusters_clients) print('number of clusters is ',n, ' silhouette score: {:<.3f}'.format(silhouette_avg)) print(pd.DataFrame(pd.Series(clusters_clients).value_counts(), columns = ['nb. de clients']).T,'\n') n_clusters_cust= 8 kmeans = KMeans(init='k-means++', n_clusters = n_clusters_cust, n_init=100) kmeans.fit(scaled_matrix) clusters_clients = kmeans.predict(scaled_matrix) silhouette_avg = silhouette_score(scaled_matrix, clusters_clients) print('silhouette score: {:<.3f}'.format(silhouette_avg)) """ At first, I look at the number of customers in each cluster: """ pd.DataFrame(pd.Series(clusters_clients).value_counts(), columns = ['nb. de clients']).T """ ** a / _Report via the PCA_ ** There is a certain disparity in the sizes of different groups that have been created. Hence I will now try to understand the content of these clusters in order to validate (or not) this particular separation. At first, I use the result of the PCA: """ pca = PCA(n_components=6) matrix_3D = pca.fit_transform(scaled_matrix) mat = pd.DataFrame(matrix_3D) mat['cluster'] = pd.Series(clusters_clients) """ in order to create a representation of the various clusters: """ import matplotlib.patches as mpatches sns.set_style("white") sns.set_context("notebook", font_scale=1, rc={"lines.linewidth": 2.5}) LABEL_COLOR_MAP = {0:'r', 1:'tan', 2:'b', 3:'k', 4:'c', 5:'g', 6:'deeppink', 7:'skyblue', 8:'darkcyan', 9:'orange', 10:'yellow', 11:'tomato', 12:'seagreen'} label_color = [LABEL_COLOR_MAP[l] for l in mat['cluster']] fig = plt.figure(figsize = (12,10)) increment = 0 for ix in range(6): for iy in range(ix+1, 6): increment += 1 ax = fig.add_subplot(4,3,increment) ax.scatter(mat[ix], mat[iy], c= label_color, alpha=0.5) plt.ylabel('PCA {}'.format(iy+1), fontsize = 12) plt.xlabel('PCA {}'.format(ix+1), fontsize = 12) ax.yaxis.grid(color='lightgray', linestyle=':') ax.xaxis.grid(color='lightgray', linestyle=':') ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) if increment == 12: break if increment == 12: break #_______________________________________________ # I set the legend: abreviation -> airline name comp_handler = [] for i in range(n_clusters): comp_handler.append(mpatches.Patch(color = LABEL_COLOR_MAP[i], label = i)) plt.legend(handles=comp_handler, bbox_to_anchor=(1.1, 0.9), title='Cluster', shadow = True, frameon = True, framealpha = 1, fontsize = 13, bbox_transform = plt.gcf().transFigure) plt.tight_layout() """ From this representation, it can be seen, for example, that the first principal component allow to separate the tiniest clusters from the rest. More generally, we see that there is always a representation in which two clusters will appear to be distinct. ** b/ _Score de silhouette intra-cluster_ ** As with product categories, another way to look at the quality of the separation is to look at silouhette scores within different clusters: """ sample_silhouette_values = silhouette_samples(scaled_matrix, clusters_clients) #____________________________________ # define individual silouhette scores sample_silhouette_values = silhouette_samples(scaled_matrix, clusters_clients) #__________________ # and do the graph graph_component_silhouette(n_clusters_cust, [-0.15, 0.55], len(scaled_matrix), sample_silhouette_values, clusters_clients) """ ** c/ _Customers morphotype_** At this stage, I have verified that the different clusters are reasonably disjoint (at least, in a global way). It remains to understand the behavior of the customers in each cluster. To do so, I start by adding to the `selected_customers` dataframe a variable that defines the cluster to which each client belongs: """ # We draw the TSNE plot to visualize the eight customer clusters. plt.figure(figsize=(20,15)) tsne = TSNE(n_components=2, verbose=1, perplexity=50, n_iter=500) tsne_ft = tsne.fit_transform(scaled_matrix) km_data = pd.DataFrame(kmeans.predict(scaled_matrix), columns=['y']) km_data['tsne-1'] = tsne_ft[:, 0] km_data['tsne-2'] = tsne_ft[:, 1] sns.scatterplot(x="tsne-1", y="tsne-2",hue=km_data["y"].tolist(), palette=sns.color_palette("hls", 8), data=km_data, legend="full", alpha=0.5) selected_customers.loc[:, 'cluster'] = clusters_clients """ Then, I average the contents of this dataframe by first selecting the different groups of clients. This gives access to, for example, the average baskets price, the number of visits or the total sums spent by the clients of the different clusters. I also determine the number of clients in each group (variable ** size **): """ merged_df = pd.DataFrame() for i in range(n_clusters_cust): test = pd.DataFrame(selected_customers[selected_customers['cluster'] == i].mean()) test = test.T.set_index('cluster', drop = True) test['size'] = selected_customers[selected_customers['cluster'] == i].shape[0] merged_df = pd.concat([merged_df, test]) #_____________________________________________________ merged_df.drop('CustomerID', axis = 1, inplace = True) print('number of customers:', merged_df['size'].sum()) merged_df = merged_df.sort_values('sum') """ Finally, I re-organize the content of the dataframe by ordering the different clusters: first, in relation to the amount wpsent in each product category and then, according to canceled%, number of unique products, and the total amount spent: """ liste_index = [] focused_cols=['categ_0', 'categ_1', 'categ_2', 'categ_3', 'categ_4', 'categ_5', 'canceled%', 'unique_prod','sum'] for col in focused_cols: idx=merged_df[col].idxmax() if idx not in liste_index: liste_index.append(idx) liste_index_reordered = liste_index.copy() liste_index_reordered += [ s for s in merged_df.index if s not in liste_index] merged_df = merged_df.reindex(index = liste_index_reordered) merged_df = merged_df.reset_index(drop = False) merged_df['cluster']=merged_df['cluster'].astype(int) merged_df """ ** d / _Customers morphology_ ** Finally, I created a representation of the different morphotypes. To do this, I define a class to create "Radar Charts" (which has been adapted from this [kernel](https://www.kaggle.com/yassineghouzam/don-t-know-why-employees-leave -read-this)): """ def _scale_data(data, ranges): (x1, x2) = ranges[0] d = data[0] return [(d - y1) / (y2 - y1) * (x2 - x1) + x1 for d, (y1, y2) in zip(data, ranges)] class RadarChart(): def __init__(self, fig, location, sizes, variables, ranges, n_ordinate_levels = 6): angles = np.arange(0, 360, 360./len(variables)) ix, iy = location[:] ; size_x, size_y = sizes[:] axes = [fig.add_axes([ix, iy, size_x, size_y], polar = True, label = "axes{}".format(i)) for i in range(len(variables))] _, text = axes[0].set_thetagrids(angles, labels = variables) for txt, angle in zip(text, angles): if angle > -1 and angle < 181: txt.set_rotation(angle - 90) else: txt.set_rotation(angle - 270) for ax in axes[1:]: ax.patch.set_visible(False) ax.xaxis.set_visible(False) ax.grid("off") for i, ax in enumerate(axes): grid = np.linspace(*ranges[i],num = n_ordinate_levels) grid_label = [""]+["{:.0f}".format(x) for x in grid[1:-1]] ax.set_rgrids(grid, labels = grid_label, angle = angles[i]) ax.set_ylim(*ranges[i]) self.angle = np.deg2rad(np.r_[angles, angles[0]]) self.ranges = ranges self.ax = axes[0] def plot(self, data, *args, **kw): sdata = _scale_data(data, self.ranges) self.ax.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kw) def fill(self, data, *args, **kw): sdata = _scale_data(data, self.ranges) self.ax.fill(self.angle, np.r_[sdata, sdata[0]], *args, **kw) def legend(self, *args, **kw): self.ax.legend(*args, **kw) def title(self, title, *args, **kw): self.ax.text(0.9, 1, title, transform = self.ax.transAxes, *args, **kw) """ This allows to have a global view of the content of each cluster: """ # fig = plt.figure(figsize=(10,8)) # attributes = ['count', 'mean', 'sum', 'canceled%','unique_prod', # 'categ_0', 'categ_1', 'categ_2', 'categ_3', 'categ_4','categ_5'] # ranges = [[0.01, 10], [0.01, 1500], [0.01, 10000], [0.01, 15], [0.01,100], # [0.01, 50], [0.01, 50], [0.01, 50], [0.01, 50], [0.01, 50], [0.01, 50]] # index = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] # n_groups = n_clusters_cust ; i_cols = 3 # i_rows = n_groups//i_cols # size_x, size_y = (1/i_cols), (1/i_rows) # for ind in range(n_clusters_cust): # ix = ind%3 ; iy = i_rows - ind//3 # pos_x = ix*(size_x + 0.05) ; pos_y = iy*(size_y + 0.05) # location = [pos_x, pos_y] ; sizes = [size_x, size_y] # #______________________________________________________ # data = np.array(merged_df.loc[index[ind], attributes]) # radar = RadarChart(fig, location, sizes, attributes, ranges) # radar.plot(data, color = 'b', linewidth=2.0) # radar.fill(data, alpha = 0.2, color = 'b') # radar.title(title = 'cluster nº{}\ncnt:{}'.format(index[ind],merged_df.loc[index[ind], "size"]), color = 'r') # ind += 1 """ It can be seen, for example, that the first 5 clusters correspond to a strong preponderance of purchases in a particular category of products. Other clusters will differ from basket averages (**mean**), the total sum spent by the clients (**sum**) or the total canceled value as a percent of total spent (**canceled%**). """ """ # 5. Make Customer Value Measures """ cvm=transactions_per_user[['sum','count','max','unique_prod','canceled%']].copy() cvm.rename(columns={'count':'num_visits','max':'max_order'},inplace=True) print("Number of customers:",len(cvm)) cvm.describe() def f_score(x,mid,c,max_f): return 1+(max_f-1)/(1+math.e**(-(x-mid)/c)) var='num_visits' plt.figure(figsize=(10,5)) plt.hist(cvm[var],bins=20) # cvm[var].hist(bins=20) plt.xlabel(var,fontsize=15) plt.ylabel('Count',fontsize=15) plt.show() var='num_visits' mid=9 c=1.5 max_f=1.3 cvm['f1']=cvm[var].apply(lambda x:f_score(x,mid,c,max_f)) plt.figure(figsize=(10,5)) plt.scatter(data=cvm,x=var,y='f1') plt.xlabel(var,fontsize=15) plt.ylabel('f1',fontsize=15) # cvm.plot.scatter(var,'f1') plt.show() var='max_order' plt.figure(figsize=(10,5)) plt.hist(cvm[var],bins=50) plt.xlabel(var,fontsize=15) plt.ylabel('Count',fontsize=15) plt.show() var='max_order' mid=900 c=180 max_f=1.1 cvm['f2']=cvm[var].apply(lambda x:f_score(x,mid,c,max_f)) plt.figure(figsize=(10,5)) plt.scatter(data=cvm,x=var,y='f2') plt.xlabel(var,fontsize=15) plt.ylabel('f2',fontsize=15) plt.show() var='unique_prod' plt.figure(figsize=(10,5)) plt.hist(cvm[var],bins=50) plt.xlabel(var,fontsize=15) plt.ylabel('Count',fontsize=15) plt.show() var='unique_prod' mid=130 c=25 max_f=1.2 cvm['f3']=cvm[var].apply(lambda x:f_score(x,mid,c,max_f)) plt.figure(figsize=(10,5)) plt.scatter(data=cvm,x=var,y='f3') plt.xlabel(var,fontsize=15) plt.ylabel('f3',fontsize=15) plt.show() var='canceled%' plt.figure(figsize=(10,5)) plt.hist(cvm[var],bins=15) plt.xlabel(var,fontsize=15) plt.ylabel('Logscale of Count',fontsize=15) plt.yscale('log') plt.show() cvm['CVM']=cvm['sum']*cvm['f1']*cvm['f2']*cvm['f3'] cvm.describe() xhigh=8000 temp=cvm[cvm['CVM']<xhigh]['CVM'] plt.figure(figsize=(15,8)) plt.hist(temp,bins=50,edgecolor='blue') plt.yscale('log') plt.ylim(1,2000) plt.xlim(-350,xhigh) plt.axvline(cvm['CVM'].quantile(0.25),color='black',linewidth=4) plt.axvline(cvm['CVM'].quantile(0.5),color='black',linewidth=4) plt.axvline(cvm['CVM'].quantile(0.75),color='black',linewidth=4) plt.annotate(f"Group C\n <{cvm['CVM'].quantile(0.25):.1f}", (-300,1100),fontweight='bold',fontsize=15) plt.annotate(f"Group B\n>{cvm['CVM'].quantile(0.5):.1f}", (720,1100),fontweight='bold',fontsize=15) plt.annotate(f"Group A\n>{cvm['CVM'].quantile(0.75):,.1f}", (1780,1100),fontweight='bold',fontsize=15) plt.xlabel('CVM',fontsize=18) plt.ylabel('Log Scale of Count',fontsize=18) plt.savefig('value_groups.png') plt.show() """ It can be seen, for example, that the first 5 clusters correspond to a strong