from collections import defaultdict
import streamlit as st
from utils import load_and_preprocess_data
import pandas as pd
import numpy as np
import altair as alt
from sklearn.mixture import GaussianMixture
import plotly.express as px
import itertools
from typing import Dict, List, Tuple
SIDEBAR_DESCRIPTION = """
# Client clustering
To cluster a client, we adopt the RFM metrics. They stand for:
- R = recency, that is the number of days since the last purchase
in the store
- F = frequency, that is the number of times a customer has ordered something
- M = monetary value, that is how much a customer has spent buying
from your business.
Given these 3 metrics, we can cluster the customers and find a suitable
"definition" based on the clusters they belong to. Since the dataset
we're using right now has about 5000 distinct customers, we identify
3 clusters for each metric.
## How we compute the clusters
We resort to a GaussianMixture algorithm. We can think of GaussianMixture
as generalized k-means clustering that incorporates information about
the covariance structure of the data as well as the centers of the clusters.
""".lstrip()
FREQUENCY_CLUSTERS_EXPLAIN = """
The **frequency** denotes how frequently a customer has ordered.
There 3 available clusters for this metric:
- cluster 1: denotes a customer that purchases one or few times (range [{}, {}])
- cluster 2: these customer have a discrete amount of orders (range [{}, {}])
- cluster 3: these customer purchases lots of times (range [{}, {}])
-------
""".lstrip()
RECENCY_CLUSTERS_EXPLAIN = """
The **recency** refers to how recently a customer has bought;
There 3 available clusters for this metric:
- cluster 1: the last order of these client is long time ago (range [{}, {}])
- cluster 2: these are clients that purchases something not very recently (range [{}, {}])
- cluster 3: the last order of these client is a few days/weeks ago (range [{}, {}])
-------
""".lstrip()
MONETARY_CLUSTERS_EXPLAIN = """
The **revenue** refers to how much a customer has spent buying
from your business.
There 3 available clusters for this metric:
- cluster 1: these clients spent little money (range [{}, {}])
- cluster 2: these clients spent a considerable amount of money (range [{}, {}])
- cluster 3: these clients spent lots of money (range [{}, {}])
-------
""".lstrip()
EXPLANATION_DICT = {
"Frequency_cluster": FREQUENCY_CLUSTERS_EXPLAIN,
"Recency_cluster": RECENCY_CLUSTERS_EXPLAIN,
"Revenue_cluster": MONETARY_CLUSTERS_EXPLAIN,
}
def create_features(df: pd.DataFrame):
"""Creates a new dataframe with the RFM features for each client."""
# Compute frequency, the number of distinct time a user purchased.
client_features = df.groupby("CustomerID")["InvoiceDate"].nunique().reset_index()
client_features.columns = ["CustomerID", "Frequency"]
# Add monetary value, the total revenue for each single user.
client_takings = df.groupby("CustomerID")["Price"].sum()
client_features["Revenue"] = client_takings.values
# Add recency, i.e. the days since the last purchase in the store.
max_date = df.groupby("CustomerID")["InvoiceDate"].max().reset_index()
max_date.columns = ["CustomerID", "LastPurchaseDate"]
client_features["Recency"] = (
max_date["LastPurchaseDate"].max() - max_date["LastPurchaseDate"]
).dt.days
return client_features
@st.cache
def cluster_clients(df: pd.DataFrame):
"""Computes the RFM features and clusters for each user based on the RFM metrics."""
df_rfm = create_features(df)
for to_cluster, order in zip(
["Revenue", "Frequency", "Recency"], ["ascending", "ascending", "descending"]
):
kmeans = GaussianMixture(n_components=3, random_state=42)
labels = kmeans.fit_predict(df_rfm[[to_cluster]])
df_rfm[f"{to_cluster}_cluster"] = _order_cluster(kmeans, labels, order)
return df_rfm
def _order_cluster(cluster_model: GaussianMixture, clusters, order="ascending"):
"""Orders the cluster by `order`."""
centroids = cluster_model.means_.sum(axis=1)
if order.lower() == "descending":
centroids *= -1
ascending_order = np.argsort(centroids)
lookup_table = np.zeros_like(ascending_order)
# Cluster will start from 1
lookup_table[ascending_order] = np.arange(cluster_model.n_components) + 1
return lookup_table[clusters]
def show_purhcase_history(user: int, df: pd.DataFrame):
user_purchases = df.loc[df.CustomerID == user, ["Price", "InvoiceDate"]]
expenses = user_purchases.groupby(user_purchases.InvoiceDate).sum()
expenses.columns = ["Expenses"]
expenses = expenses.reset_index()
c = (
alt.Chart(expenses)
.mark_line(point=True)
.encode(
x=alt.X("InvoiceDate", timeUnit="yearmonthdate", title="Date"),
y="Expenses",
)
.properties(title="User expenses")
)
st.altair_chart(c)
def show_user_info(user: int, df_rfm: pd.DataFrame):
"""Prints some information about the user.
The main information are the total expenses, how
many times he purchases in the store, and the clusters
he belongs to.
"""
user_row = df_rfm[df_rfm["CustomerID"] == user]
if len(user_row) == 0:
st.write(f"No user with id {user}")
output = []
output.append(f"The user purchased **{user_row['Frequency'].squeeze()} times**.\n")
output.append(
f"She/he spent **{user_row['Revenue'].squeeze()} dollars** in total.\n"
)
output.append(
f"The last time she/he bought something was **{user_row['Recency'].squeeze()} days ago**.\n"
)
output.append(f"She/he belongs to the clusters: ")
for cluster in [column for column in user_row.columns if "_cluster" in column]:
output.append(f"- {cluster} = {user_row[cluster].squeeze()}")
st.write("\n".join(output))
return (
user_row["Recency_cluster"].squeeze(),
user_row["Frequency_cluster"].squeeze(),
user_row["Revenue_cluster"].squeeze(),
)
def explain_cluster(cluster_info):
"""Displays a popup menu explinging the meanining of the clusters."""
with st.expander("Show information about the clusters"):
st.write(
"**Note**: these values are valid for these dataset."
"Different dataset will have different number of clusters"
" and values"
)
for cluster, info in cluster_info.items():
# Transform the (mins, maxs) tuple into
# [min_1, max_1, min_2, max_2, ...] list.
min_max_interleaved = list(itertools.chain(*zip(info[0], info[1])))
st.write(EXPLANATION_DICT[cluster].format(*min_max_interleaved))
def categorize_user(recency_cluster, frequency_cluster, monetary_cluster):
"""Describe the user with few words based on the cluster he belongs to."""
score = f"{recency_cluster}{frequency_cluster}{monetary_cluster}"
# @fixme: find a better approeach. These elif chains don't scale at all.
description = ""
if score == "111":
description = "Tourist"
elif score.startswith("2"):
description = "Losing interest"
elif score == "133":
description = "Former lover"
elif score == "123":
description = "Former passionate client"
elif score == "113":
description = "Spent a lot, but never come back"
elif score.startswith("1"):
description = "About to dump"
elif score == "313":
description = "Potential lover"
elif score == "312":
description = "Interesting new client"
elif score == "311":
description = "New customer"
elif score == "333":
description = "Gold client"
elif score == "322":
description = "Lovers"
else:
description = "Average client"
st.write(f"The customer can be described as: **{description}**")
def plot_rfm_distribution(
df_rfm: pd.DataFrame, cluster_info: Dict[str, Tuple[List[int], List[int]]]
):
"""Plots 3 histograms for the RFM metrics."""
for x, to_reverse in zip(("Revenue", "Frequency", "Recency"), (False, False, True)):
fig = px.histogram(
df_rfm,
x=x,
log_y=True,
title=f"{x} metric",
)
# Get the max value in the cluster info. The cluster_info_dict is a
# tuple with first element the min values of the cluster, and second
# element the max values of the cluster.
values = cluster_info[f"{x}_cluster"][1] # get max values
print(values)
# Add vertical bar on each cluster end. But skip the last cluster.
loop_range = range(len(values) - 1)
if to_reverse:
# Skip the last element
loop_range = range(len(values) - 1, 0, -1)
for n_cluster in loop_range:
print(x)
print(values[n_cluster])
fig.add_vline(
x=values[n_cluster],
annotation_text=f"End of cluster {n_cluster+1}",
line_dash="dot",
annotation=dict(textangle=90, font_color="red"),
)
fig.update_layout(
yaxis_title="Count (log scale)",
)
st.plotly_chart(fig)
def display_dataframe_heatmap(df_rfm: pd.DataFrame, cluster_info_dict):
"""Displays an heatmap of how many clients lay in the clusters.
This method uses some black magic coming from the dataframe
styling guide.
"""
def style_with_limits(x, column, cluster_limit_dict):
"""Simple function to transform the cluster number into
a cluster + range string."""
min_v = cluster_limit_dict[column][0][x - 1]
max_v = cluster_limit_dict[column][1][x - 1]
return f"{x}: [{int(min_v)}, {int(max_v)}]"
# Create a dataframe with the count of clients for each group
# of cluster.
count = (
df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
"CustomerID"
]
.count()
.reset_index()
)
count = count.rename(columns={"CustomerID": "Count"})
# Remove duplicates
count = count.drop_duplicates(
["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]
)
# Add limits to the cells. In this way, we can better display
# the heatmap.
for cluster in ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]:
count[cluster] = count[cluster].apply(
lambda x: style_with_limits(x, cluster, cluster_info_dict)
)
# Use the count column as values, then index with the clusters.
count = count.pivot(
index=["Revenue_cluster", "Frequency_cluster"],
columns="Recency_cluster",
values="Count",
)
# Style manipulation
cell_hover = {
"selector": "td",
"props": "font-size:1.2em",
}
index_names = {
"selector": ".index_name",
"props": "font-style: italic; color: Black; font-weight:normal;font-size:1.2em;",
}
headers = {
"selector": "th:not(.index_name)",
"props": "background-color: White; color: black; font-size:1.2em",
}
# Finally, display
# We cannot directly print the dataframe since the streamlit
# functin remove the multiindex. Thus, we extract the html representation
# and then display it.
st.markdown("## Heatmap: how the client are distributed between clusters")
st.write(
count.style.format(thousands=" ", precision=0, na_rep="0")
.set_table_styles([cell_hover, index_names, headers])
.background_gradient(cmap="coolwarm")
.to_html(),
unsafe_allow_html=True,
)
def main():
st.sidebar.markdown(SIDEBAR_DESCRIPTION)
df, _, _ = load_and_preprocess_data()
df_rfm = cluster_clients(df)
st.markdown(
"# Dataset "
"\nThis is the processed dataset with information about the clients, such as"
" the RFM values and the clusters they belong to."
)
st.dataframe(df_rfm.style.format(formatter={"Revenue": "{:.2f}"}))
cluster_info_dict = defaultdict(list)
with st.expander("Show more details about the clusters"):
for cluster in [column for column in df_rfm.columns if "_cluster" in column]:
st.write(cluster)
cluster_info = (
df_rfm.groupby(cluster)[cluster.split("_")[0]]
.describe()
.reset_index(names="Cluster")
)
min_cluster = cluster_info["min"].astype(int)
max_cluster = cluster_info["max"].astype(int)
cluster_info_dict[cluster] = (min_cluster, max_cluster)
st.dataframe(cluster_info)
st.markdown("## RFM metric distribution")
plot_rfm_distribution(df_rfm, cluster_info_dict)
display_dataframe_heatmap(df_rfm, cluster_info_dict)
st.markdown("## Interactive exploration")
filter_by_cluster = st.checkbox(
"Filter client: only one client per cluster type",
value=True,
)
client_to_select = (
df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
"CustomerID"
]
.first()
.values
if filter_by_cluster
else df["CustomerID"].unique()
)
# Let the user select the user to investigate
user = st.selectbox(
"Select a customer to show more information about him.",
client_to_select,
)
show_purhcase_history(user, df)
recency, frequency, revenue = show_user_info(user, df_rfm)
categorize_user(recency, frequency, revenue)
explain_cluster(cluster_info_dict)
main()