# Uncomment to install required python modules
# !sh ../utils/setup.sh

# Add custom utils module to Python environment
import os
import sys
sys.path.append(os.path.abspath(os.pardir))

import numpy as np
import pandas as pd

from gps_building_blocks.analysis.exp_design import ab_testing_design
from gps_building_blocks.cloud.utils import bigquery as bigquery_utils

from utils import helpers

"""
Explanation: 6. Media experiment design
This notebook demonstrates the design of a media experiment by using the
Experimental Desing
module to activate the predictions from a propensity model. It is vital to design and estimate the impact of media campaigns using valid statistical methods to make sure the limited experimentation budget is utilized effectively and to set the right expectations of the campaign outcome.
Requirements

An already scored test dataset, or the model and the test dataset to be scored available in GCP BigQuery.
This test dataset should contain all the ML instances for at least one snapshot date.

Install and import required modules
End of explanation
"""


# Prints all the outputs from cell (instead of using display each time).
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

"""
Explanation: Notebook custom settings
End of explanation
"""


configs = helpers.get_configs('config.yaml')
dest_configs, run_id_configs = configs.destination, configs.run_id

# GCP project ID
PROJECT_ID = dest_configs.project_id
# Name of the BigQuery dataset
DATASET_NAME = dest_configs.dataset_name

# To distinguish the separate runs of the training pipeline
RUN_ID = run_id_configs.train

# BigQuery table name containing the test dataset to be scored. This test
# dataset should contain all the instances at least for one snapshot date
FEATURES_TEST_TABLE = f'features_test_table_{RUN_ID}'
# BigQuery model name
MODEL_NAME = f'propensity_model_{RUN_ID}'
# BigQuery table name containing the scored test dataset.
FEATURES_TEST_PREDICTIONS_TABLE = f'features_test_table_preds_{RUN_ID}'
# Selected snapshot date to select the ML instances (reflecting the instances to
# be scored on a given scoring date) to be used for experiment design in
# YYYY-MM-DD format
SELECTED_SNAPSHOT_DATE = '2017-06-15'
# Name of the actual label column
ACTUAL_LABEL_NAME = 'label'
# Name of the prediction column
PREDICTED_LABEL_NAME = 'predicted_label_probs'
# Label value for the positive class
POSITIVE_CLASS_LABEL = True

# BigQuery client object
bq_utils = bigquery_utils.BigQueryUtils(project_id=PROJECT_ID)

"""
Explanation: Set parameters
End of explanation
"""


# Prediction sql query
# TODO(): Filter deliberately data for ML.Predict
prediction_query =f"""
  SELECT *
  FROM ML.PREDICT(MODEL `{PROJECT_ID}.{DATASET_NAME}.{MODEL_NAME}`,
                  TABLE `{PROJECT_ID}.{DATASET_NAME}.{FEATURES_TEST_TABLE}`)
                  WHERE snapshot_ts='{SELECTED_SNAPSHOT_DATE}';
"""

# Run prediction
print(prediction_query)
df_test_predictions = bq_utils.run_query(prediction_query).to_dataframe()

# Size of the prediction data frame
print(df_test_predictions.shape)

"""
Explanation: Select the relevant data for experiment design
Select all the instances for one snapshot date, which resembles the scoring dataset for one day. This dataset is used to design the media experiment.
Score the test dataset (if not already scored)
End of explanation
"""


# Data read in sql query
read_query = f"""
SELECT
  predictions.label AS label,
  predicted_label,
  probs.label AS predicted_score_label,
  probs.prob AS score,
  snapshot_ts
FROM
  `{PROJECT_ID}.{DATASET_NAME}.{FEATURES_TEST_PREDICTIONS_TABLE}` AS predictions,
  UNNEST({PREDICTED_LABEL_NAME}) AS probs
WHERE
  probs.label={POSITIVE_CLASS_LABEL}
  AND snapshot_ts = '{SELECTED_SNAPSHOT_DATE}';
"""

# Run prediction
print(read_query)
df_test_predictions = bq_utils.run_query(read_query).to_dataframe()

# Size of the prediction data frame
print(df_test_predictions.shape)

"""
Explanation: Read the prediction test dataset (if already scored)
End of explanation
"""


df_test_predictions.head()

# Change positive label into 1.0.
df_test_predictions['label_numerical'] = [
                          1.0 if label == POSITIVE_CLASS_LABEL else 0.0
                          for label in df_test_predictions[ACTUAL_LABEL_NAME]]

# Check transformed values
label_before = df_test_predictions['label'].value_counts()
label_after = df_test_predictions['label_numerical'].value_counts()
df_labels_check = pd.DataFrame(data = {'label_before': label_before,
                     'label_after': label_after})
df_labels_check['is_label_same'] = (
    df_labels_check['label_before'] == df_labels_check['label_after'])
df_labels_check

"""
Explanation: Prepare probability and label columns
End of explanation
"""


ab_testing_design.calc_chisquared_sample_sizes_for_bins(
    labels=df_test_predictions['label_numerical'].values,
    probability_predictions=df_test_predictions['score'].values,
    number_bins=3, # to have High, Medium and Low bins
    uplift_percentages=(10, 15), # minimum expected effect sizes
    power_percentages=(80, 90),
    confidence_level_percentages=(90, 95))

"""
Explanation: Experiment Design I: Different Propensity Groups
One way to use the output from a Propensity Model to optimize marketing is to first define different audience groups based on the predicted probabilities (such as High, Medium and Low propensity groups) and then test the same or different marketing strategies with those. This strategy is more useful to understand how different propensity groups respond to remarketing campaigns.
Following step estimates the statistical sample sizes required for different groups (bins) of the predicted probabilities based on different combinations of the expected minimum uplift/effect size, statistical power and statistical confidence levels specified as input parameters.
Expected output: a Pandas Dataframe containing statistical sample size for each bin for each combination of minimum uplift_percentage, statistical power and statistical confidence level.
Based on the estimated sample sizes and the available sizes one can decide what setting (expected minimum uplift/effect size at a given statistical power and a confidence level) to be selected for the experiment. Then the selected sample sizes could be used to set Test and Control cohorts from each propensity group to implement the media experiment.
End of explanation
"""


ab_testing_design.calc_chisquared_sample_sizes_for_cumulative_bins(
    labels=df_test_predictions['label_numerical'].values,
    probability_predictions=df_test_predictions['score'].values,
    number_bins=10, # top 10%, 20%, ..., 100%
    uplift_percentages=(10, 15), # minimum expected effect sizes
    power_percentages=(80, 90),
    confidence_level_percentages=(90, 95))

"""
Explanation: Experiment Design II: Top Propensity Group
Another way to use the output from a Propensity Model to optimize marketing is to target the top X% of users having the highest predicted probability score in a remarketing campaign or an acquisition campaigns with the similar audience strategy.
Following step estimates the statistical sample sizes required for different cumulative groups (bins) of the predicted probabilities (top X%, top 2X% and so on) based on different combinations of the expected minimum uplift/effect size, statistical power and statistical confidence levels specified as input parameters.
Expected output: a Pandas Dataframe containing statistical sample size for each cumulative bin for each combination of minimum uplift_percentage, statistical power and statistical confidence level.
Based on the estimated sample sizes and the available sizes one can decide what setting (what top X% of users with the expected minimum uplift/effect size at a given statistical power and a confidence level) to be selected for the experiment. Then the selected sample size could be used to set Test and Control cohorts from the top X% to implement the media experiment.
End of explanation
"""


