# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Input space analysis
#
# We briefly analyze potential correlations in customers' attribute space. This will help us to define the right set of features to plug into the analysis. We perform the analysis for a small number of campaigns.
#
# 1. [Cinema Christmas](#Cinema-Christmas)
# 2. [Mobile](#Mobile)
# 3. [TV Winback UK](#TV-Winback-UK)

# +
import numpy as np
import pandas as pd

import logging
import os
import gcsfs
import time
import datetime
import dateutil.relativedelta
from google.cloud import bigquery

import matplotlib.pyplot as plt
import seaborn as sns

import sys
sys.path.append('../../../variable-clustering/')
sys.path.append('../../../STLP/')
sys.path.append('{}/decisioning-analysis/test_and_learn/production_modules/'.format(os.environ['HOME']))

from varclushi import VarClusHi
from correlation.cramersv import CramersV

from preprocessing_utils.campaign_info_preprocessing import get_campaigns_relevant_info
from preprocessing_utils.campaign_info_preprocessing import create_right_granularity_level
from preprocessing_utils.campaign_info_preprocessing import create_prop_mart_date
from preprocessing_utils.campaign_info_preprocessing import provide_target_flag

from io_utils.input_extraction import pull_table
from preprocessing_utils.input_features_preprocessing import preprocess_data

from analysis_execution.run_ccp_analysis import create_logger

# +
CAMPAIGN_DATA_PATHS = {'project_id':'skyuk-uk-decis-models-01-dev',
                       'dataset_id':'test_and_learn',
                       'table_id':'optimus_base_clean'}
PROP_MART_DATA_PATHS = {'project_id':'skyuk-uk-decis-models-01-dev',
                       'dataset_id':'uk_inp_prodrecs_ic',
                       'table_id':'Propensity_Model_Mart_Training'} 

OPTIMUS_COLS = {'program_label':'L1_programme',
                'campaign_label':'L2_Campaign',
                'journey_label':'L3_Journey',
                'step_label':'L4_Step',
                'variant_label':'L5_variant',
                'product_label':'product',
                'account_number_label':'account_number',
                'sale_vol_label':'sale_vol',
                'sale_aux_vol_label':'sale_auxillary_vol',
                'cell_label':'cell_type',
                'control_label':'control_type2',
                'start_date_label':'CAMPAIGN_REPORTING_START_DATE',
                'channel_label':'CHANNEL'}

PROPMART_FEAT = ['Base_Dt',
                 'account_number',
                 
                 'TAs_in_last_14d',
                 'BB_Curr_Contract_Intended_End_Dt',
                 'DTV_Curr_Contract_Intnd_End_Dt',
                 'Curr_Offer_Intended_end_Dt_DTV',
                 'Curr_Offer_Intended_end_Dt_BB',
                 'BB_Last_Activation_Dt',
                 'DTV_Last_Activation_Dt',
                 
                 'DTV_Active',
                 'BB_Active',
                 'Sports_Active',
                 'Movies_Active',
                 'MS_Active',
                 'SGE_Active',
                 'HD_Active',
                 'Sky_Box_Sets_Active',
                 'UOD_Active',
                 'SkyKids_Active',
                 'Handset_Active',
                 'skyfibre_enabled',
                 
                 'Age',
                 'h_equivalised_income_value',
                 'h_residence_type',
                 'h_number_of_adults',
                 'h_number_of_children_in_hh',
                 'h_presence_of_child_aged_0_4',
                 'h_presence_of_child_aged_5_11',
                 'h_presence_of_child_aged_12_17',
                 
                 'Broadband_Average_Demand',
                 'BT_Consumer_Market_Share',
                 'Sky_Consumer_Market_Share',
                 'Virgin_Consumer_Market_Share',
                 'TalkTalk_Consumer_Market_Share',
                 
                 
                 'Cricket_Sports_Pref',
                 'Football_Sports_Pref',
                 'Rugby_Union_Sports_Pref',
                 'Rugby_League_Sports_Pref',
                 'F1_Sports_Pref',
                 'Darts_Sports_Pref',
                 'Tennis_Sports_Pref',
                 'Racing_Sports_Pref',
                 'Boxing_Sports_Pref',
                 'Golf_Sports_Pref',
                 'NFL_Sports_Pref',
                 'Wrestling_Sports_Pref',
                 'ESports_Sports_Pref',
                 'GAA_Sports_Pref',
                 'Motorsport_Sports_Pref',
                 'Cycling_Sports_Pref',
                 'Netball_Sports_Pref',

                 'Up_Handset',
                 'Up_Mobile ']

BUCKET_ID = 'fci02/test_and_learn'

mapping_dict = {'h_presence_of_child_aged_0_4':{'No':0,'Yes':1},
                'h_presence_of_child_aged_5_11':{'No':0,'Yes':1},
                'h_presence_of_child_aged_12_17':{'No':0,'Yes':1},}

tree_pms = {'n_estimators':[500],
            'min_samples_leaf':[500],
            'max_depth':[ 5],
            'max_allowed_depth':5,
            'min_allowed_node_size':5000,
            'n_best_features':3,
            'njobs':14}

control_name_dict = {'skyOptimusGetMore':'Campaign control','skyOptimusDMO':'Fallow Group'}

PROJECT_ID_1 = 'skyuk-uk-decis-models-01-dev'
DATASET_ID_1 = 'test_and_learn'
BUCKET_ID = 'fci02/test_and_learn'
CAMPAING_TABLE_NAME = 'optimus_base_clean'

granularity_level = 'overall'

base_date_label='Base_Dt'
default_treat_table_name='Campaign_treat_tmp'
control_name_dict = {'skyOptimusGetMore':'Campaign control','skyOptimusDMO':'Fallow Group'}

to_exclude = ['Base_Dt', 'account_number','L2_Campaign','target']
# -

campaigns_list = get_campaigns_relevant_info(PROJECT_ID_1,DATASET_ID_1,CAMPAING_TABLE_NAME,
                                             OPTIMUS_COLS['program_label'],
                                             OPTIMUS_COLS['campaign_label'],
                                             OPTIMUS_COLS['start_date_label'], 
                                             OPTIMUS_COLS['product_label'],
                                             OPTIMUS_COLS['journey_label'],
                                             OPTIMUS_COLS['step_label'],
                                             OPTIMUS_COLS['variant_label']
                                            )
campaigns_list.head(10)

# ## Cinema Christmas

to_analyze = campaigns_list[['campaign_program','campaign_name']].drop_duplicates()
idx=to_analyze[to_analyze.campaign_name=='Cinema Christmas'].index
to_analyze.loc[idx,['campaign_program','campaign_name']]

granularity_level_table,analysis_level = create_right_granularity_level(campaigns_list,to_analyze.loc[idx,['campaign_program','campaign_name']], granularity_level)
prop_mart_date = create_prop_mart_date(granularity_level_table)
granularity_level_table
subset_row = granularity_level_table.loc[0,:]
target_var_list = provide_target_flag(subset_row)

test_table = pull_table(CAMPAIGN_DATA_PATHS, PROP_MART_DATA_PATHS, subset_row,target_var_list,analysis_level,
                        OPTIMUS_COLS, PROPMART_FEAT,
                        base_date_label, prop_mart_date,subset_row['start_date'],
                        '=', control_name_dict[subset_row['campaign_program']], subset_row['product'],default_treat_table_name,
                        BUCKET_ID)

test_table.head()

test_table.shape

logger = create_logger()
_ = preprocess_data(test_table,mapping_dict,to_exclude,logger=logger)

test_table.shape


def is_binary(series, allow_na=False):
    if allow_na:
        series.dropna(inplace=True)
    return sorted(series.unique()) == [0, 1]


numeric = test_table.drop(columns=to_exclude)._get_numeric_data().columns
for column in numeric:
    if is_binary(test_table[column], allow_na=False):
        print(column)

binary = ['DTV_Active', 'BB_Active', 'Sports_Active', 'Movies_Active', 'MS_Active' ,'SGE_Active', 'HD_Active', 'Sky_Box_Sets_Active', 'UOD_Active', 'SkyKids_Active',
          'Handset_Active', 'h_presence_of_child_aged_0_4', 'h_presence_of_child_aged_5_11', 'h_presence_of_child_aged_12_17', 'Up_Handset', 'Up_Mobile']

real_numeric = [feature for feature in numeric if feature not in binary]

analyzer = VarClusHi(test_table[real_numeric],maxeigval2=1,maxclus=None)

analyzer.varclus()

analyzer.rsquare

# Cluster 2 and 4 are the really interesting ones

test_table[['Age','h_number_of_adults','tenure_months']].corr()

sns.pairplot(test_table[['Age','h_number_of_adults','tenure_months']])
plt.show()

test_table[['h_equivalised_income_value','h_number_of_children_in_hh']].corr()

sns.pairplot(test_table[['h_equivalised_income_value','h_number_of_children_in_hh']])
plt.show()

categorical = [column for column in test_table.drop(columns=to_exclude).columns if column not in real_numeric]
categorical

cramer_analyzer = CramersV(test_table[categorical])

cramer_table=cramer_analyzer.produce_correlation_table()

plt.figure(figsize=(20,10))
cramer_analyzer.produce_heatmap(cramer_table)

# ## Mobile

to_analyze = campaigns_list[['campaign_program','campaign_name']].drop_duplicates()
idx=to_analyze[to_analyze.campaign_name=='Mobile'].index
to_analyze.loc[idx,['campaign_program','campaign_name']]

granularity_level_table,analysis_level = create_right_granularity_level(campaigns_list,to_analyze.loc[idx,['campaign_program','campaign_name']], granularity_level)
prop_mart_date = create_prop_mart_date(granularity_level_table)
granularity_level_table
subset_row = granularity_level_table.loc[0,:]
target_var_list = provide_target_flag(subset_row)

test_table = pull_table(CAMPAIGN_DATA_PATHS, PROP_MART_DATA_PATHS, subset_row,target_var_list,analysis_level,
                        OPTIMUS_COLS, PROPMART_FEAT,
                        base_date_label, prop_mart_date,subset_row['start_date'],
                        '=', control_name_dict[subset_row['campaign_program']], subset_row['product'],default_treat_table_name,
                        BUCKET_ID)

test_table.head()

test_table.shape

logger = create_logger()
_ = preprocess_data(test_table,mapping_dict,to_exclude,logger=logger)

test_table.shape


def is_binary(series, allow_na=False):
    if allow_na:
        series.dropna(inplace=True)
    return sorted(series.unique()) == [0, 1]


numeric = test_table.drop(columns=to_exclude)._get_numeric_data().columns
for column in numeric:
    if is_binary(test_table[column], allow_na=False):
        print(column)

binary = ['DTV_Active', 'BB_Active', 'Sports_Active', 'Movies_Active', 'MS_Active' ,'SGE_Active', 'HD_Active', 'Sky_Box_Sets_Active', 'UOD_Active', 'SkyKids_Active',
          'Handset_Active', 'h_presence_of_child_aged_0_4', 'h_presence_of_child_aged_5_11', 'h_presence_of_child_aged_12_17', 'Up_Handset', 'Up_Mobile']

real_numeric = [feature for feature in numeric if feature not in binary]

analyzer = VarClusHi(test_table[real_numeric],maxeigval2=1,maxclus=None)

analyzer.varclus()

analyzer.rsquare

# Cluster 2 and 4 are the really interesting ones

test_table[['Age','h_number_of_adults','tenure_months']].corr()

sns.pairplot(test_table[['Age','h_number_of_adults','tenure_months']])
plt.show()

test_table[['h_equivalised_income_value','h_number_of_children_in_hh']].corr()

sns.pairplot(test_table[['h_equivalised_income_value','h_number_of_children_in_hh']])
plt.show()

categorical = [column for column in test_table.drop(columns=to_exclude).columns if column not in real_numeric]
categorical

cramer_analyzer = CramersV(test_table[categorical])

cramer_table=cramer_analyzer.produce_correlation_table()

plt.figure(figsize=(20,10))
cramer_analyzer.produce_heatmap(cramer_table)

# ## TV Winback UK

to_analyze = campaigns_list[['campaign_program','campaign_name']].drop_duplicates()
idx=to_analyze[to_analyze.campaign_name=='Winback DTV UK'].index
to_analyze.loc[idx,['campaign_program','campaign_name']]

granularity_level_table,analysis_level = create_right_granularity_level(campaigns_list,to_analyze.loc[idx,['campaign_program','campaign_name']], granularity_level)
prop_mart_date = create_prop_mart_date(granularity_level_table)
granularity_level_table
subset_row = granularity_level_table.loc[0,:]
target_var_list = provide_target_flag(subset_row)

test_table = pull_table(CAMPAIGN_DATA_PATHS, PROP_MART_DATA_PATHS, subset_row,target_var_list,analysis_level,
                        OPTIMUS_COLS, PROPMART_FEAT,
                        base_date_label, prop_mart_date,subset_row['start_date'],
                        '=', control_name_dict[subset_row['campaign_program']], subset_row['product'],default_treat_table_name,
                        BUCKET_ID)

test_table.head()

test_table.shape

logger = create_logger()
_ = preprocess_data(test_table,mapping_dict,to_exclude,logger=logger)

test_table.shape


def is_binary(series, allow_na=False):
    if allow_na:
        series.dropna(inplace=True)
    return sorted(series.unique()) == [0, 1]


numeric = test_table.drop(columns=to_exclude)._get_numeric_data().columns
for column in numeric:
    if is_binary(test_table[column], allow_na=False):
        print(column)

binary = ['DTV_Active', 'BB_Active', 'Sports_Active', 'Movies_Active', 'MS_Active' ,'SGE_Active', 'HD_Active', 'Sky_Box_Sets_Active', 'UOD_Active', 'SkyKids_Active',
          'Handset_Active', 'h_presence_of_child_aged_0_4', 'h_presence_of_child_aged_5_11', 'h_presence_of_child_aged_12_17', 'Up_Handset', 'Up_Mobile']

real_numeric = [feature for feature in numeric if feature not in binary]

analyzer = VarClusHi(test_table[real_numeric],maxeigval2=1,maxclus=None)

analyzer.varclus()

analyzer.rsquare

# Cluster 2 and 4 are the really interesting ones

test_table[['Age','h_number_of_adults','tenure_months']].corr()

sns.pairplot(test_table[['Age','h_number_of_adults','tenure_months']])
plt.show()

test_table[['h_equivalised_income_value','h_number_of_children_in_hh']].corr()

sns.pairplot(test_table[['h_equivalised_income_value','h_number_of_children_in_hh']])
plt.show()

categorical = [column for column in test_table.drop(columns=to_exclude).columns if column not in real_numeric]
categorical

cramer_analyzer = CramersV(test_table[categorical])

cramer_table=cramer_analyzer.produce_correlation_table()

plt.figure(figsize=(20,10))
cramer_analyzer.produce_heatmap(cramer_table)

# ## Summer Cinema

to_analyze = campaigns_list[['campaign_program','campaign_name']].drop_duplicates()
idx=to_analyze[to_analyze.campaign_name=='Summer Cinema'].index
to_analyze.loc[idx,['campaign_program','campaign_name']]

granularity_level_table,analysis_level = create_right_granularity_level(campaigns_list,to_analyze.loc[idx,['campaign_program','campaign_name']], granularity_level)
prop_mart_date = create_prop_mart_date(granularity_level_table)
granularity_level_table
subset_row = granularity_level_table.loc[0,:]
target_var_list = provide_target_flag(subset_row)

test_table = pull_table(CAMPAIGN_DATA_PATHS, PROP_MART_DATA_PATHS, subset_row,target_var_list,analysis_level,
                        OPTIMUS_COLS, PROPMART_FEAT,
                        base_date_label, prop_mart_date,subset_row['start_date'],
                        '=', control_name_dict[subset_row['campaign_program']], subset_row['product'],default_treat_table_name,
                        BUCKET_ID)

test_table.head()

test_table.shape

logger = create_logger()
_ = preprocess_data(test_table,mapping_dict,to_exclude,logger=logger)

test_table.shape

numeric = test_table.drop(columns=to_exclude)._get_numeric_data().columns
for column in numeric:
    if is_binary(test_table[column], allow_na=False):
        print(column)

binary = ['DTV_Active', 'BB_Active', 'Sports_Active', 'Movies_Active', 'MS_Active' ,'SGE_Active', 'HD_Active', 'Sky_Box_Sets_Active', 'UOD_Active', 'SkyKids_Active',
          'Handset_Active', 'h_presence_of_child_aged_0_4', 'h_presence_of_child_aged_5_11', 'h_presence_of_child_aged_12_17', 'Up_Handset', 'Up_Mobile']

real_numeric = [feature for feature in numeric if feature not in binary]

analyzer = VarClusHi(test_table[real_numeric],maxeigval2=1,maxclus=None)

analyzer.varclus()

analyzer.rsquare

# Cluster 2 and 4 are the really interesting ones

test_table[['Age','h_number_of_adults','tenure_months','ncomms']].corr()

sns.pairplot(test_table[['Age','h_number_of_adults','tenure_months','ncomms']])
plt.show()

test_table[['h_equivalised_income_value','h_number_of_children_in_hh']].corr()

sns.pairplot(test_table[['h_equivalised_income_value','h_number_of_children_in_hh']])
plt.show()

categorical = [column for column in test_table.drop(columns=to_exclude).columns if column not in real_numeric]
categorical

cramer_analyzer = CramersV(test_table[categorical])

cramer_table=cramer_analyzer.produce_correlation_table()

plt.figure(figsize=(20,10))
cramer_analyzer.produce_heatmap(cramer_table)

cramer_table[['BB_contract_status', 'DTV_contract_status']]



StarcoderdataJupyterScriptsDedupFiltered