# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="vmpUSz8cPM3C" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1009} outputId="ba37d7d0-f304-4ca3-f42a-4953a4e61100" executionInfo={"status": "ok", "timestamp": 1549782661206, "user_tz": -60, "elapsed": 19239, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
# !pip install catboost
# !pip install tsfresh
# !pip install xgboost

# + id="kSDbdv4nPliC" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 74} outputId="5ef2233c-6bf6-4726-910e-e7bcbc239d09" executionInfo={"status": "ok", "timestamp": 1549782662245, "user_tz": -60, "elapsed": 20270, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
# The essentials
import pandas as pd
import numpy as np

from collections import defaultdict

# Plotting
# %matplotlib inline
import matplotlib.pyplot as plt

# Progress bars
from tqdm import tqdm

# Access our Google Drive
from google.colab import drive

# Gradient Boosting
from catboost import CatBoostRegressor, Pool
from xgboost import XGBRegressor

# TSFRESH Feature Extraction
from tsfresh import extract_features
from tsfresh.feature_extraction import EfficientFCParameters
from tsfresh.utilities.dataframe_functions import impute
from tsfresh.feature_selection.relevance import calculate_relevance_table

from sklearn.model_selection import KFold

from collections import defaultdict, Counter
from scipy.stats import norm

# + id="YwLXNiaTPmtm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 5033} outputId="52faf555-b775-46cf-bea4-3a79bc18ada0" executionInfo={"status": "ok", "timestamp": 1549782761265, "user_tz": -60, "elapsed": 112840, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
drive.mount('/content/drive', force_remount=True)
# !ls "/content/drive/My Drive/Rinse Over Run"

# + id="2ttT2U5bPoWd" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 54} outputId="6300d33a-f563-4525-cc2c-0b325a715deb" executionInfo={"status": "ok", "timestamp": 1549782936275, "user_tz": -60, "elapsed": 80298, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
train_df = pd.read_csv('/content/drive/My Drive/Rinse Over Run/train_values.csv', index_col=0, parse_dates=['timestamp'])
test_df = pd.read_csv('/content/drive/My Drive/Rinse Over Run/test_values.csv', index_col=0, parse_dates=['timestamp'])
label_df = pd.read_csv('/content/drive/My Drive/Rinse Over Run/train_labels.csv', index_col='process_id')
all_data = pd.concat([train_df, test_df], axis=0)

# + id="aR5gDVrWW6qJ" colab_type="code" colab={}
train_df = train_df[train_df['phase'] != 'final_rinse']

train_df['phase_int'] = train_df['phase'].map({'pre_rinse': 1, 
                                               'caustic': 2, 
                                               'intermediate_rinse': 4, 
                                               'acid': 8})
test_df['phase_int'] = test_df['phase'].map({'pre_rinse': 1, 
                                             'caustic': 2, 
                                             'intermediate_rinse': 4, 
                                             'acid': 8})
train_process_combinations = pd.DataFrame(train_df.groupby('process_id')['phase_int'].unique().apply(lambda x: sum(x)))
test_process_combinations = pd.DataFrame(test_df.groupby('process_id')['phase_int'].unique().apply(lambda x: sum(x)))
process_combinations = pd.concat([train_process_combinations, test_process_combinations], axis=0)

recipe_df = pd.read_csv('/content/drive/My Drive/Rinse Over Run/recipe_metadata.csv', index_col='process_id')
recipe_df = recipe_df.drop('final_rinse', axis=1)
recipe_df['pre_rinse_num'] = recipe_df['pre_rinse'] * 1
recipe_df['caustic_num'] = recipe_df['caustic'] * 2
recipe_df['intermediate_rinse_num'] = recipe_df['intermediate_rinse'] * 4
recipe_df['acid_num'] = recipe_df['acid'] * 8
recipe_df['recipe'] = recipe_df['pre_rinse_num'] + recipe_df['caustic_num'] + recipe_df['intermediate_rinse_num'] + recipe_df['acid_num']

# + id="tjoO0VcrRMvA" colab_type="code" colab={}
ts_real = [
    'supply_flow',
    'supply_pressure',
    'return_temperature',
    'return_conductivity',
    'return_turbidity',
    'return_flow',
    'tank_level_pre_rinse',
    'tank_level_caustic',
    'tank_level_acid',
    'tank_level_clean_water',
    'tank_temperature_pre_rinse',
    'tank_temperature_caustic',
    'tank_temperature_acid',
    'tank_concentration_caustic',
    'tank_concentration_acid',
    #'target_value',
    #'flow_diff'
]

# variables we'll use to create our time series features
ts_cols = [
    'supply_flow',
    'supply_pressure',
    'return_temperature',
    'return_conductivity',
    'return_turbidity',
    'return_flow',
    'tank_level_pre_rinse',
    'tank_level_caustic',
    'tank_level_acid',
    'tank_level_clean_water',
    'tank_temperature_pre_rinse',
    'tank_temperature_caustic',
    'tank_temperature_acid',
    'tank_concentration_caustic',
    'tank_concentration_acid',
    #'target_value',
    #'flow_diff'
]

# variables for binary time series features
bin_cols = [
    'supply_pump',
    'supply_pre_rinse',
    'supply_caustic',
    'return_caustic',
    'supply_acid',
    'return_acid',
    'supply_clean_water',
    'return_recovery_water',
    'return_drain',
    'object_low_level',
    'tank_lsh_caustic',
    'tank_lsh_acid',
    'tank_lsh_clean_water',
    'tank_lsh_pre_rinse'
]

process_comb_to_phases = {
    15: ['pre_rinse', 'caustic', 'intermediate_rinse', 'acid'],
    3:  ['pre_rinse', 'caustic'],
    7:  ['pre_rinse', 'caustic', 'intermediate_rinse'],
    1:  ['pre_rinse'],
    8:  ['acid'],
    2:  ['caustic'],
    6:  ['caustic', 'intermediate_rinse'],
    14: ['caustic', 'intermediate_rinse', 'acid'],
}

# phases, ordered from earliest to latest
phases = ['pre_rinse', 'caustic', 'intermediate_rinse', 'acid']

def get_descript(data, functions, cols):
    ts_df = data.set_index('process_id').sort_values(by='timestamp')
    return ts_df.groupby('process_id')[cols].agg(functions)  

  
def get_descript_prev_process(data, data_procs):
    machines = set(data['object_id'])
    all_features = []
    for machine in tqdm(machines):
        machine_data = data[data['object_id'] == machine]
        machine_data = machine_data.sort_values(by='timestamp')
        machine_processes = machine_data['process_id'].unique()
        for process_ix, process in enumerate(machine_processes):
          if process in data_procs:
            if process_ix > 0:
                prev_process = machine_data[machine_data['process_id'] == machine_processes[process_ix - 1]]
                last_phase = None
                for phase in phases:
                  if phase in set(prev_process['phase']):
                    last_phase = phase
                prev_process = prev_process[prev_process['phase'] == last_phase]
                
                features = get_descript(prev_process, ['mean', 'std', 'min', 'max', 'count'], ts_cols)
                _columns = list(features.columns)
                assert len(features) == 1
                features = features.iloc[0, :].values
                
                this_process = machine_data[machine_data['process_id'] == machine_processes[process_ix]]
                prev_recipe = recipe_df.loc[machine_processes[process_ix - 1]]['recipe']
                prev_phase_int = process_combinations.loc[machine_processes[process_ix - 1]]['phase_int']
                time_delta = (this_process['timestamp'].values[0] - prev_process['timestamp'].values[-1]) / np.timedelta64(1, 'h')
                assert time_delta > 0
                all_features.append([machine, process, time_delta, prev_recipe, prev_phase_int] + list(features))
            else:
                all_features.append([machine, process, np.NaN, np.NaN, np.NaN]  + ([np.NaN] * 60))
   
    all_features = pd.DataFrame(all_features, columns=['object_id', 'process_id', 'time_delta', 'prev_recipe', 'prev_phase_int'] + _columns)
    all_features = all_features.set_index('process_id', drop=True)
    col_map = {}
    for col in all_features.columns:
        col_map[col] = 'prev_{}'.format(col)
    all_features = all_features.rename(columns=col_map)
    all_features = all_features.drop('prev_object_id', axis=1)
    return all_features
  
def get_processes(data, phases, train=True):
    filtered_processes = []
    phases = set(phases)
    processes = set(data['process_id'])
    for process in processes:
        process_phases = set(data[data['process_id'] == process]['phase'])
        if train:
            if phases.issubset(process_phases):
                filtered_processes.append(process)
        else:
            if len(phases) == len(process_phases) == len(phases.intersection(process_phases)):
                filtered_processes.append(process)
    return filtered_processes


# + id="VyH4tLQNQbBm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1174} outputId="ddc0b2b6-d64f-4067-aae8-ae310416506b" executionInfo={"status": "ok", "timestamp": 1549787684592, "user_tz": -60, "elapsed": 318, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
sub_df = all_data[all_data['object_id'] == 945]
sub_df.groupby('process_id')['timestamp'].max().sort_values()

# + id="xzgboa9_RULH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="6b788802-4a24-4000-d531-b297d7676e03" executionInfo={"status": "ok", "timestamp": 1549788248565, "user_tz": -60, "elapsed": 3127, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}
features = get_descript_prev_process(sub_df, list(set(sub_df['process_id'])))

# + id="Imt38iEHRxvQ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 2061} outputId="b8bcc67f-4361-460a-dc3a-b8294e1679fa" executionInfo={"status": "ok", "timestamp": 1549788248572, "user_tz": -60, "elapsed": 2999, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "04070140729205161481"}}

features

# + id="n-yIRfy6R9RH" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered