# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import numpy as np
from datetime import date

# Data preprocessing
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder

# Data split
from sklearn.model_selection import train_test_split
# -

# # Data exploration

# In this section I will be exploring the dataset in regards of different aspects such as the completeness, the data type, and more. This will guide me in the data cleaning process

# Loading the data
mist_data = pd.read_csv("../data/Mistplay_challenge_data_science.csv") 

# Analyzing each of the variables in the dataset
mist_data.describe()

# Data type and missing values
mist_data.info()

mist_data.columns

# ## Variable Exploration

# This section will help me learning about the target and the different variables the data has

categorical_features = ['Phone OS version',
       'Phone device model', 'Countries','User source', 'Year of birth', 'User attribute 10']

# This section counts the number of observations per level of each variable
for feat in categorical_features:
    print('Feature: %s' %(feat))
    print('------------')
    print(mist_data[feat].value_counts())
    print('\n\n')

# ## Data cleaning & Feature Engineering

# In this section I am going to clean the NA values and create new variables such as the age (given that we have the year of birth)

# +
# This section converts the bi-level variable into a binary one so it is easier to model
mist_data['Age'] = date.today().year - mist_data['Year of birth']

mist_data['Phone brand'] = mist_data['Phone device model'].str.extract(r"([A-Za-z]*)_")

# Reducing the number of levels in Phone brand/user source
series = pd.value_counts(mist_data['Phone brand'])
threshold = (series<= 10)

mist_data['Phone brand sh'] = np.where(mist_data['Phone brand'].isin(series[threshold].index),'Other',mist_data['Phone brand'])

series = pd.value_counts(mist_data['User source'])
threshold = series<= 10

mist_data['User source sh'] = np.where(mist_data['User source'].isin(series[threshold].index),'Other',mist_data['User source'])

# This part will be useful later for modeling. Please, note I am not using the user ID since it is not useful for modeling
categorical_features = ['Phone OS version', 'Countries','User source sh', 'User attribute 10', 'Phone brand sh']

numeric_features = ['User attribute 1', 'User attribute 2',
       'User attribute 3', 'User attribute 4', 'User attribute 5',
       'User attribute 6', 'User attribute 7', 'User attribute 8',
       'User attribute 9', 'User attribute 11',
       'User attribute 12', 'Age']

target = ['Target variable Y']
# -

mist_data[categorical_features] = mist_data[categorical_features].fillna(value='MV')
mist_data.head()

for feat in categorical_features + target:
    print('Feature: %s' %(feat))
    print('------------')
    print(mist_data[feat].value_counts())
    print('\n\n')

# As you can see there is a huge imbalance problem

length_data = len(mist_data['Target variable Y'])
print(f"I will work with {length_data} Observations")

mist_data_chopped = mist_data[numeric_features + categorical_features + target]

# +
# I am using a train-test split of 70%
train, test = train_test_split(mist_data_chopped, train_size = 0.7, random_state = 1234)
train.to_csv("../data/processed_data/train_before-preprocessed.csv")

y_train = train[target]
y_test = test[target]

X_train = train[numeric_features + categorical_features]
X_test = test[numeric_features + categorical_features]
# -

X_train

# # Preprocessing

# This step is needed as some of the models use distance metric, and also categorical features must be turned into dummies

# For this part I am turning the categorical variables into dummy variables and the numeric variables
# into a normal distribution
preprocessor = ColumnTransformer(
    transformers=[
        ('scale', StandardScaler(), numeric_features),
        ('ohe', OneHotEncoder(handle_unknown='ignore'), categorical_features)])

# +
train = pd.DataFrame(preprocessor.fit_transform(X_train).toarray(),
                       index=X_train.index,
                       columns=(numeric_features +
                                list(preprocessor.named_transformers_['ohe']
                                     .get_feature_names(categorical_features))))


test = pd.DataFrame(preprocessor.transform(X_test).toarray(),
                      index=X_test.index,
                      columns=train.columns)
# -

train['Y'] = y_train
test['Y'] = y_test

train.head()

test.head()

# Printing the data
train.to_csv("../data/processed_data/mist_train.csv")
test.to_csv("../data/processed_data/mist_test.csv")

StarcoderdataJupyterScriptsDedupFiltered