# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/sbuttler/CTGAN/blob/master/CTGAN_newsvendor.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="wqKMQFgWUzl_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 602} outputId="983d8730-330f-4145-d24f-f3b433efc428"
# !pip install --upgrade git+git://github.com/sbuttler/CTGAN.git

# + id="fCQGAqWZXpLW" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 319} outputId="40617938-f59a-4fe1-e85f-0b5e0448d518"
# !pip install pulp

# + id="mZ4Ujx3TU7VC" colab_type="code" colab={}
import pandas as pd
import pickle
import numpy as np

import itertools
import timeit

import ctgan.synthesizer as ctgan
from sklearn.preprocessing import MinMaxScaler

import torch

#from ddop.datasets.load_datasets import load_data
#from ddop.newsvendor import EmpiricalRiskMinimizationNewsvendor
#from ddop.metrics.costs import calc_costs, calc_total_costs, calc_avg_costs

from sklearn.model_selection import train_test_split
import lightgbm as lgb

import pulp

import multiprocessing as mp
from functools import partial
import itertools

# + id="2VFDqF8nU93B" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 126} outputId="bc5180ed-1c49-4688-db11-e896a3439bba"
from google.colab import drive
drive.mount('/content/drive')

# + id="outDkMy1WeyU" colab_type="code" colab={}
model_path = '/content/drive/My Drive/GANs/'

# + id="GqdOpX2LVAi4" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="801fbc83-3662-44a0-aaa0-9d672d33503a"
demand = 'demand'

train = pd.read_pickle('/content/drive/My Drive/GANs/Data/20200629_train_FilNr2_ArtNr110_all_cols')
train.drop(columns=['Filiale', 'Artikel', 'WGruppe', 'time_index'], axis=1, inplace=True)
X_train = train.drop(columns=[demand])
Y_train = train[demand]

val = pd.read_pickle('/content/drive/My Drive/GANs/Data/20200629_val_FilNr2_ArtNr110_all_cols')
val.drop(columns=['Filiale', 'Artikel', 'WGruppe', 'time_index'], axis=1, inplace=True)
X_val = val.drop(columns=[demand])
Y_val = val[demand]

test = pd.read_pickle('/content/drive/My Drive/GANs/Data/20200629_test_FilNr2_ArtNr110_all_cols')
test.drop(columns=['Filiale', 'Artikel', 'WGruppe', 'time_index'], axis=1, inplace=True)
X_test = test.drop(columns=[demand])
Y_test = test[demand]

scaler = MinMaxScaler()
scaler.fit(train)

# + id="6wxHuN0nVoqv" colab_type="code" colab={}
# Define dataframe to store solution

service_levels = [x/100 for x in range(5, 100, 5)]

result = pd.DataFrame(index=service_levels, columns=['cost_val_solution', 'cost_test_solution', 'GAN_model_parameters'])

result['cost_val_solution'] = np.inf
result['cost_test_solution'] = np.inf

# + [markdown] id="OdeqkR6YVcW_" colab_type="text"
# # GAN parameters

# + id="PCewpQjpVSWQ" colab_type="code" colab={}
epochs = [5]
#epochs = [5, 10, 50, 100]
batchsize = [30, 60, 130]
learning_rate = [1e-5, 5e-5, 1e-6]

training_repetition = 1


# + [markdown] id="7Yz_bHt6ViVH" colab_type="text"
# # Function definitions

# + id="FbppfrlSVks5" colab_type="code" colab={}
def solve_newsvendor(demand_data, weights, sl):
    # set service level
    c_u = sl
    c_o = 1 - sl
    
    print('started newsvendor')

    sample_index = np.arange(demand_data.shape[0])
    
    demand = dict(zip(sample_index, demand_data))
    weights = dict(zip(sample_index, weights))

    m = pulp.LpProblem(sense=pulp.LpMinimize)

    # add variables
    q = pulp.LpVariable('q', lowBound=0)

    u = pulp.LpVariable.dicts('u', sample_index, lowBound=0)
    o = pulp.LpVariable.dicts('o', sample_index, lowBound=0)


    # set objective

    m += (sum(c_u*u[i]*weights[i] for i in sample_index) + sum(c_o * o[i]*weights[i] for i in sample_index))

    # add constraints
    for i in sample_index:

      m += u[i] >= demand[i] - q
      m += o[i] >= q - demand[i]
    
    m.solve()
    
    return q.value()


# + id="JIytiaHOVsNY" colab_type="code" colab={}
def compute_lightGBM_wSAA_weights(sl, gbm, X_train, Y_train, val_x):
    
    print("compute light GBM wSAA")
    # Predict test data
    predLeafs = gbm.predict(val_x, pred_leaf=True)

    # Calculate weights based on training data
    predLeafsTrain = gbm.predict(X_train, pred_leaf=True)
    elemInLeafs = np.sum(predLeafsTrain == predLeafs, axis=0)
    treeWeights = (predLeafsTrain == predLeafs)/elemInLeafs
    weights = np.sum(treeWeights, axis=1)/predLeafs.shape[1]
    
    # Optimize
    q = solve_newsvendor(Y_train, weights, sl)

    return q


# + id="q8xMgoQJVwin" colab_type="code" colab={}
def compute_lightGBM_wSAA(sl, train_X, train_Y, eval_X, eval_Y, test_eval_X):
    
    
    print('fit lightGBM model')
    
    # fit lightGBM model
    trainingData = lgb.Dataset(train_X, train_Y)
    evalData = lgb.Dataset(eval_X, eval_Y)

    param = {'objective': 'regression'}

    num_round = 1000

    # fitting model 
    gbm = lgb.train(
        param,
        trainingData,
        num_round,
        valid_sets=evalData,
        early_stopping_rounds=10,
        verbose_eval=100
    )
    
       
    wSAA_function = partial(compute_lightGBM_wSAA_weights, sl, gbm, train_X, train_Y)
    
    q = test_eval_X.apply(wSAA_function, axis=1)
    
    return q


# + id="Kd7WCZ8NV44-" colab_type="code" colab={}
def newsvendor_total_cost(y_pred, y_true, sl):
    
    total_cost = 0
    
    # set service level
    c_u = sl
    c_o = 1 - sl
    
    for i in range(y_pred.shape[0]):
        
        if y_pred[i] > y_true[i]:
            
            total_cost += (y_pred[i]-y_true[i])*c_o
            
        if y_pred[i] < y_true[i]:
            
            total_cost += (y_true[i]-y_pred[i])*c_u
    
    return total_cost


# + id="jelQuS4xV_5N" colab_type="code" colab={}
def lightGBM_wSAA_cost(gan_params, train_X, train_Y, eval_X, eval_Y, test_eval_X, test_eval_Y, sl):
    
    y_pred = compute_lightGBM_wSAA(sl, train_X, train_Y, eval_X, eval_Y, test_eval_X)
    
    y_true = test_eval_Y

    y_pred.reset_index(inplace=True, drop=True)
    y_true.reset_index(inplace=True, drop=True)
    
    return (sl, [newsvendor_total_cost(y_pred, y_true, sl), gan_params])


# + [markdown] id="BpBDlYgVWWeI" colab_type="text"
# # Test with one GAN and small sample size

# + id="1lopbbQQWVv-" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="140386e2-7588-4c1c-8134-7f55ecbcb19f"
service_levels = [0.7]

e = 5
bs = 30
lr = 1e-5
i = 1

p_gan = str(i)+"_e"+str(e)+"_b"+str(bs)+"_l"+str(lr)

gan = torch.load(model_path+"ctgan_model_"+str(i)+"_e"+str(e)+"_b"+str(bs)+"_l"+str(lr))

print('sample')
sample_sc = gan.sample(1000000, seed=0)
sample = pd.DataFrame(scaler.inverse_transform(sample_sc), columns=sample_sc.columns)

print('finished')
X = sample.drop(columns=[demand])
Y = sample[demand]

X_train_gan, X_val_gan, Y_train_gan, Y_val_gan = train_test_split(X, Y, test_size=0.2, random_state=99)
        
pool_function = partial(lightGBM_wSAA_cost, p_gan, X_train_gan, Y_train_gan,  X_val_gan, Y_val_gan, X_val, Y_val)

print('open pool')
#agents = mp.cpu_count()
agents = len(service_levels)
pool = mp.Pool(agents)

pool_output = pool.map(pool_function, service_levels)

pool.close()

pool_output = dict(pool_output)

print(pool_output)

for sl in service_levels:
      
  val_cost = pool_output.get(sl)[0]
  parameters = pool_output.get(sl)[1]
  
  if val_cost < result.loc[sl,'cost_val_solution']:
        
    result.loc[sl,'cost_val_solution'] = val_cost
    result.loc[sl, 'GAN_model_parameters'] = parameters
    print('test')


# + id="yM64z-HZe05t" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 625} outputId="4f55459d-c946-488d-de8c-2622992b6fda"
result

# + [markdown] id="6jJocVtHWSjX" colab_type="text"
# # Full GAN newsvendor loop

# + id="wH1NngjHWIi2" colab_type="code" colab={}
for lr in learning_rate:

  for bs in batchsize:

    for e in epochs:

      for i in range(1,training_repetition+1):
        
        p_gan = (lr,bs,e,i)
        
        gan = torch.load(model_path+"ctgan_model_"+str(i)+"_e"+str(e)+"_b"+str(bs)+"_l"+str(lr))
        
        print('sample')
        sample_sc = gan.sample(1000000, seed=0)
        sample = pd.DataFrame(scaler.inverse_transform(sample_sc), columns=sample_sc.columns)
        
        print('finished')
        X = sample.drop(columns=[demand])
        Y = sample[demand]
        
        X_train_gan, X_val_gan, Y_train_gan, Y_val_gan = train_test_split(X, Y, test_size=0.2, random_state=99)
                
        pool_function = partial(ERM_cost, p_gan, X_train_gan, Y_train_gan, X_val, Y_val)

        print('open pool')
        agents = mp.cpu_count()
        pool = mp.Pool(agents)
        
        pool_output = pool.map(pool_function, service_levels)

        pool.close()

        pool_output = dict(pool_output)
        
        for sl in service_levels:
      
          val_cost = pool_output.get(sl)[0]
          parameters = pool_output.get(sl)[1]
          
          if val_cost < result.loc[sl,'cost_val_solution']:
                
            result.loc[sl,'cost_val_solution'] = val_cost
            result.loc[sl, 'GAN_model_parameters'] = parameters
            print('test')


StarcoderdataJupyterScriptsDedupFiltered