# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Building and fitting NN

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
import sys
import pandas as pd
import numpy as np
import scipy.sparse as sparse
from scipy.sparse.linalg import spsolve
import random
import os
import scipy.stats as ss
import scipy
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from catboost import CatBoostClassifier, Pool, sum_models, to_classifier
from sklearn.model_selection import KFold
from sklearn.utils.class_weight import compute_class_weight

import implicit

# +
#df = pd.read_csv("../input/feature-creating-v2/train_3lags_v3.csv", low_memory=False)

# +
#df = df.loc[df['order_count'] == 2]

# +
#df.head()

# +
#threshold = 0.0005
#counts = df['service_title'].value_counts(normalize=True)
#df2 = df.loc[df['service_title'].isin(counts[counts > threshold].index), :]

# +
#sub15 = pd.read_csv("../input/catboost-fitting-smart/submission_15.csv")

# +
#d_134 = df2.loc[df2['service_title'] == 134][:121500]

# +
#d_98 = df2.loc[df2['service_title'] == 98][:121500]

# +
#df2 = df2.loc[df2['service_title'] != 134]
#df2 = df2.loc[df2['service_title'] != 98]

# +
#df2 = pd.concat([df2,d_134], axis=0)
#df2 = pd.concat([df2,d_98], axis=0)

# +
#df2.to_csv('train_3lags_semibalanced.csv', index=False)
# -

# Downloading prepared data

df2 = pd.read_csv("../input/irkutsk/train_3lags_semibalanced.csv", low_memory=False)

# +
#df2 = pd.read_csv("../input/irkutsk/train_3lags_v4.csv", low_memory=False)

# +
#df2.drop(['Unnamed: 0'], axis=1, inplace=True)
# -

df2['service_title'].value_counts(normalize=True)[:30]

# +
#sub = pd.read_csv("../input/nn-sub/nn_sub_5fold_2.csv")

# +
#df2['service_title'].value_counts(normalize=True)[:30]

# +
#sub['service_title'] = 1259

# +
#sub.to_csv('check_1259.csv', index=False)

# +
#sub['service_title'].value_counts(normalize=True)[:30]
# -

df2 = df2.sample(frac=1).reset_index(drop=True)

df_test = df2[['service_title', 'order_count']]

df2.drop(['service',
'service_title',
'mfc',
'internal_status',
'external_status',
'order_type',
'department_id',
'custom_service_id',
'service_level',
'is_subdep',
'is_csid',
'proc_time',
'dayofweek',
'day_part',
'person',
'sole',
'legal',
'auto_ping_queue',
'win_count',
'month',
'week',
'year'], axis=1 , inplace=True)

df_train = df2.loc[df2['order_count'] > 1]

df_test = df_test.loc[df_test['order_count'] > 1]

df_train.reset_index(inplace=True)
df_test.reset_index(inplace=True)

df_train.drop(['index'], axis=1, inplace=True)
df_test.drop(['index'], axis=1, inplace=True)

# +
#mask = (df_train_mask['order_count'] == 5)
#z_valid = df_train_mask[mask]

#df_train_mask.loc[mask, 'service_4'] = np.nan

# -

df_train.columns

categorical = ['service_1', 'service_title_1', 'mfc_1',
       'internal_status_1', 'external_status_1', 'order_type_1',
       'department_id_1', 'custom_service_id_1', 'service_level_1',
       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',
       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',
       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',
       'external_status_2', 'order_type_2', 'department_id_2',
       'custom_service_id_2', 'service_level_2', 'is_subdep_2', 'is_csid_2',
       'dayofweek_2', 'day_part_2', 'person_2', 'sole_2', 'month_2', 'week_2', 'year_2',
       'legal_2', 'auto_ping_queue_2','service_3',
       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',
       'order_type_3', 'department_id_3', 'custom_service_id_3',
       'service_level_3', 'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',
       'dayofweek_3', 'day_part_3', 'person_3', 'sole_3', 'legal_3',
       'auto_ping_queue_3',
       'requester_type', 'gender']

cat = ['service_1', 'service_title_1', 'mfc_1',
       'internal_status_1', 'external_status_1',
       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',
       'is_subdep_1', 'is_csid_1', 'dayofweek_1',
       'service_2', 'service_title_2', 'mfc_2', 'internal_status_2',
       'external_status_2', 'department_id_2', 'month_2', 'week_2', 'year_2',
       'custom_service_id_2', 'is_subdep_2', 'is_csid_2',
       'dayofweek_2', 'service_3',
       'service_title_3', 'mfc_3', 'internal_status_3', 'external_status_3',
       'department_id_3', 'custom_service_id_3',
       'is_subdep_3', 'is_csid_3', 'month_3', 'week_3', 'year_3',
       'dayofweek_3', 
       'requester_type', 'gender']

X = df_train.drop(['requester'], axis=1)
y = df_test['service_title']

# +
X.drop(['service_2',
'service_title_2',
'mfc_2',
'internal_status_2',
'external_status_2',
'order_type_2',
'department_id_2',
'custom_service_id_2',
'service_level_2',
'is_subdep_2',
'is_csid_2',
'proc_time_2',
'dayofweek_2',
'day_part_2',
'person_2',
'sole_2',
'legal_2',
'auto_ping_queue_2',
'win_count_2',
'month_2',
'week_2',
'year_2'], axis=1 , inplace=True)

X.drop(['service_3',
'service_title_3',
'mfc_3',
'internal_status_3',
'external_status_3',
'order_type_3',
'department_id_3',
'custom_service_id_3',
'service_level_3',
'is_subdep_3',
'is_csid_3',
'proc_time_3',
'dayofweek_3',
'day_part_3',
'person_3',
'sole_3',
'legal_3',
'auto_ping_queue_3',
'win_count_3',
'month_3',
'week_3',
'year_3'], axis=1 , inplace=True)

# +
categorical = ['service_1', 'service_title_1', 'mfc_1',
       'internal_status_1', 'external_status_1', 'order_type_1',
       'department_id_1', 'custom_service_id_1', 'service_level_1',
       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 'day_part_1', 'month_1', 'week_1', 'year_1',
       'person_1', 'sole_1', 'legal_1', 'auto_ping_queue_1',
       'requester_type', 'gender']

cat = ['service_1', 'service_title_1', 'mfc_1',
       'internal_status_1', 'external_status_1',
       'department_id_1', 'custom_service_id_1', 'month_1', 'week_1', 'year_1',
       'is_subdep_1', 'is_csid_1', 'dayofweek_1', 
       'requester_type', 'gender']
# -

X[cat] = X[cat].astype('Int64')
X[cat] = X[cat].astype('object')

# +
#X[cat] = X[cat].astype('Int64')
#X[cat] = X[cat].astype('object')
# -

X[categorical] = X[categorical].fillna('NA')


# Function to reduce ram usage

def reduce_mem_usage(df):
    """ 
    iterate through all the columns of a dataframe and 
    modify the data type to reduce memory usage.        
    """
    start_mem = df.memory_usage().sum() / 1024**2
    print(('Memory usage of dataframe is {:.2f}' 
                     'MB').format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max <\
                  np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max <\
                   np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max <\
                   np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max <\
                   np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max <\
                   np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max <\
                   np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')
    end_mem = df.memory_usage().sum() / 1024**2
    print(('Memory usage after optimization is: {:.2f}' 
                              'MB').format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) 
                                             / start_mem))
    
    return df


X = reduce_mem_usage(X)

X = X[categorical]

X['person_1'] = X['person_1'].astype('int32')
X['sole_1'] = X['sole_1'].astype('int32')
X['legal_1'] = X['legal_1'].astype('int32')
X['auto_ping_queue_1'] = X['auto_ping_queue_1'].astype('int32')

from sklearn import preprocessing
labeling = []
for col in X.columns:
    d = pd.DataFrame()
    d[col] = X[col].unique()
    le = preprocessing.LabelEncoder()
    le.fit(d[col])
    d[col+'_l'] = le.transform(d[col])
    d = d.sort_values(by=[col+'_l']).reset_index(drop=True)
    labeling.append(d)

d = pd.DataFrame()
d['service_title'] = y.unique()
le = preprocessing.LabelEncoder()
le.fit(d['service_title'])
d['service_title'+'_l'] = le.transform(d['service_title'])
d = d.sort_values(by=['service_title'+'_l']).reset_index(drop=True)
labeling_y = d

labeling[0]

i = 0
for col in X.columns:
    X[col] = X[col].map(labeling[i].set_index(col).to_dict()[col+'_l'])
    i += 1

X

y = y.map(labeling_y.set_index('service_title').to_dict()['service_title'+'_l'])

# +
#s_w = compute_class_weight(class_weight='balanced', classes=labeling_y['service_title_l'], y=y)

weights_l = labeling_y[:]
weights_l['weights'] = 1

weights = pd.DataFrame(y)
weights = pd.merge(weights, weights_l, how='left', left_on='service_title', right_on='service_title_l')
#weights = np.array(weights['weights'])
# -

weights['service_title_y'].value_counts(normalize=True)[:30]

weights['weights'].loc[weights['service_title_y'] == 4] = 0.4
weights['weights'].loc[weights['service_title_y'] == 603] = 0.6
weights['weights'].loc[weights['service_title_y'] == 98] = 0.9
weights['weights'].loc[weights['service_title_y'] == 134] = 0.87

weights = np.array(weights['weights'])

for col in X.columns:
    print(col, len(X[col].unique()))

import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold

# +
# Detect hardware, return appropriate distribution strategy
try:
    # TPU detection. No parameters necessary if TPU_NAME environment variable is
    # set: this is always the case on Kaggle.
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    print('Running on TPU ', tpu.master())
except ValueError:
    tpu = None

if tpu:
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.
    strategy = tf.distribute.get_strategy()

print("REPLICAS: ", strategy.num_replicas_in_sync)

# +
AUTO = tf.data.experimental.AUTOTUNE

BATCH_SIZE = 64 * strategy.num_replicas_in_sync
LEARNING_RATE = 1e-3 * strategy.num_replicas_in_sync 
EPOCHS = 5
# -

X.columns

len(X['service_1'].unique())


def generate_dataset(idxT, idxV):
    
    trn_weights = weights[idxT, ]
    val_weights = weights[idxV, ]
    #n samples
    
    trn_input_ids = np.array(X.index)[idxT,]
    
    # Trainset
    trn_input_service_1 = np.array(X['service_1'])[idxT,]
    trn_input_service_title_1 = np.array(X['service_title_1'])[idxT,]
    trn_input_mfc_1 = np.array(X['mfc_1'])[idxT,]
    trn_input_internal_status_1 = np.array(X['internal_status_1'])[idxT,]
    trn_input_external_status_1 = np.array(X['external_status_1'])[idxT,]
    trn_input_order_type_1 = np.array(X['order_type_1'])[idxT,]
    trn_input_department_id_1 = np.array(X['department_id_1'])[idxT,]
    trn_input_custom_service_id_1 = np.array(X['custom_service_id_1'])[idxT,]
    trn_input_service_level_1 = np.array(X['service_level_1'])[idxT,]
    trn_input_is_subdep_1 = np.array(X['is_subdep_1'])[idxT,]
    trn_input_is_csid_1 = np.array(X['is_csid_1'])[idxT,]
    trn_input_dayofweek_1 = np.array(X['dayofweek_1'])[idxT,]
    trn_input_day_part_1 = np.array(X['day_part_1'])[idxT,]
    trn_input_month_1 = np.array(X['month_1'])[idxT,]
    trn_input_week_1 = np.array(X['week_1'])[idxT,]
    trn_input_year_1 = np.array(X['year_1'])[idxT,]
    trn_input_person_1 = np.array(X['person_1'])[idxT,]
    trn_input_sole_1 = np.array(X['sole_1'])[idxT,]
    trn_input_legal_1 = np.array(X['legal_1'])[idxT,]
    trn_input_auto_ping_queue_1 = np.array(X['auto_ping_queue_1'])[idxT,]
    trn_input_requester_type = np.array(X['requester_type'])[idxT,]
    trn_input_gender = np.array(X['gender'])[idxT,]
    
    
    trn_service_title = np.array(pd.get_dummies(y))[idxT,].astype('int32')
    
    # Validation set
    val_input_service_1 = np.array(X['service_1'])[idxV,]
    val_input_service_title_1 = np.array(X['service_title_1'])[idxV,]
    val_input_mfc_1 = np.array(X['mfc_1'])[idxV,]
    val_input_internal_status_1 = np.array(X['internal_status_1'])[idxV,]
    val_input_external_status_1 = np.array(X['external_status_1'])[idxV,]
    val_input_order_type_1 = np.array(X['order_type_1'])[idxV,]
    val_input_department_id_1 = np.array(X['department_id_1'])[idxV,]
    val_input_custom_service_id_1 = np.array(X['custom_service_id_1'])[idxV,]
    val_input_service_level_1 = np.array(X['service_level_1'])[idxV,]
    val_input_is_subdep_1 = np.array(X['is_subdep_1'])[idxV,]
    val_input_is_csid_1 = np.array(X['is_csid_1'])[idxV,]
    val_input_dayofweek_1 = np.array(X['dayofweek_1'])[idxV,]
    val_input_day_part_1 = np.array(X['day_part_1'])[idxV,]
    val_input_month_1 = np.array(X['month_1'])[idxV,]
    val_input_week_1 = np.array(X['week_1'])[idxV,]
    val_input_year_1 = np.array(X['year_1'])[idxV,]
    val_input_person_1 = np.array(X['person_1'])[idxV,]
    val_input_sole_1 = np.array(X['sole_1'])[idxV,]
    val_input_legal_1 = np.array(X['legal_1'])[idxV,]
    val_input_auto_ping_queue_1 = np.array(X['auto_ping_queue_1'])[idxV,]
    val_input_requester_type = np.array(X['requester_type'])[idxV,]
    val_input_gender = np.array(X['gender'])[idxV,]
    
    
    val_service_title = np.array(pd.get_dummies(y))[idxV,].astype('int32')
    
    # Generating tf.data object
    train_dataset = (
        tf.data.Dataset
        .from_tensor_slices(({'service_1':trn_input_service_1, 
                              'service_title_1': trn_input_service_title_1,
                              'mfc_1': trn_input_mfc_1,
                              'internal_status_1': trn_input_internal_status_1,
                              'external_status_1': trn_input_external_status_1,
                              'order_type_1': trn_input_order_type_1,
                              'department_id_1': trn_input_department_id_1,
                              'custom_service_id_1': trn_input_custom_service_id_1,
                              'service_level_1': trn_input_service_level_1,
                              'is_subdep_1': trn_input_is_subdep_1,
                              'is_csid_1': trn_input_is_csid_1,
                              'dayofweek_1': trn_input_dayofweek_1,
                              'day_part_1': trn_input_day_part_1,
                              'month_1': trn_input_month_1,
                              'week_1': trn_input_week_1,
                              'year_1': trn_input_year_1,
                              'person_1': trn_input_person_1,
                              'sole_1': trn_input_sole_1,
                              'legal_1': trn_input_legal_1,
                              'auto_ping_queue_1': trn_input_auto_ping_queue_1,
                              'requester_type': trn_input_requester_type,
                              'gender': trn_input_gender}, 
                             {'service_title': trn_service_title}, trn_weights))
        .shuffle(2048)
        .batch(BATCH_SIZE)
        .prefetch(AUTO)
    )
    
    valid_dataset = (
        tf.data.Dataset
        .from_tensor_slices(({'service_1':val_input_service_1, 
                              'service_title_1': val_input_service_title_1,
                              'mfc_1': val_input_mfc_1,
                              'internal_status_1': val_input_internal_status_1,
                              'external_status_1': val_input_external_status_1,
                              'order_type_1': val_input_order_type_1,
                              'department_id_1': val_input_department_id_1,
                              'custom_service_id_1': val_input_custom_service_id_1,
                              'service_level_1': val_input_service_level_1,
                              'is_subdep_1': val_input_is_subdep_1,
                              'is_csid_1': val_input_is_csid_1,
                              'dayofweek_1': val_input_dayofweek_1,
                              'day_part_1': val_input_day_part_1,
                              'month_1': val_input_month_1,
                              'week_1': val_input_week_1,
                              'year_1': val_input_year_1,
                              'person_1': val_input_person_1,
                              'sole_1': val_input_sole_1,
                              'legal_1': val_input_legal_1,
                              'auto_ping_queue_1': val_input_auto_ping_queue_1,
                              'requester_type': val_input_requester_type,
                              'gender': val_input_gender}, 
                             {'service_title': val_service_title}, val_weights))
        .batch(BATCH_SIZE)
        .cache()
        .prefetch(AUTO)
    )
    
    return trn_input_ids.shape[0]//BATCH_SIZE, train_dataset, valid_dataset


def scheduler(epoch):
    return LEARNING_RATE * 0.2**epoch


np.array(pd.get_dummies(y))[[0, 1],].astype('int32')


def build_model():
    service_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_1')
    service_title_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_title_1')
    mfc_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='mfc_1')
    internal_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='internal_status_1')
    external_status_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='external_status_1')
    order_type_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='order_type_1')
    department_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='department_id_1')
    custom_service_id_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='custom_service_id_1')
    service_level_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='service_level_1')
    is_subdep_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_subdep_1')
    is_csid_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='is_csid_1')
    dayofweek_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='dayofweek_1')
    day_part_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='day_part_1')
    month_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='month_1')
    week_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='week_1')
    year_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='year_1')
    person_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='person_1')
    sole_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='sole_1')
    legal_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='legal_1')
    auto_ping_queue_1 = tf.keras.layers.Input((1,), dtype=tf.int32, name='auto_ping_queue_1')
    requester_type = tf.keras.layers.Input((1,), dtype=tf.int32, name='requester_type')
    gender = tf.keras.layers.Input((1,), dtype=tf.int32, name='gender')
    
    service_1_embedding = tf.keras.layers.Embedding(len(X['service_1'].unique()), 11, input_length=1, name='service_1_embedding')(service_1)
    service_title_1_embedding = tf.keras.layers.Embedding(len(X['service_title_1'].unique()), 50, input_length=1, name='service_title_1_embedding')(service_title_1)
    mfc_1_embedding = tf.keras.layers.Embedding(len(X['mfc_1'].unique()), 50, input_length=1, name='mfc_1_embedding')(mfc_1)
    internal_status_1_embedding = tf.keras.layers.Embedding(len(X['internal_status_1'].unique()), 5, input_length=1, name='internal_status_1_embedding')(internal_status_1)
    external_status_1_embedding = tf.keras.layers.Embedding(len(X['external_status_1'].unique()), 14, input_length=1, name='external_status_1_embedding')(external_status_1)
    order_type_1_embedding = tf.keras.layers.Embedding(len(X['order_type_1'].unique()), 2, input_length=1, name='order_type_1_embedding')(order_type_1)
    department_id_1_embedding = tf.keras.layers.Embedding(len(X['department_id_1'].unique()), 46, input_length=1, name='department_id_1_embedding')(department_id_1)
    custom_service_id_1_embedding = tf.keras.layers.Embedding(len(X['custom_service_id_1'].unique()), 26, input_length=1, name='custom_service_id_1_embedding')(custom_service_id_1)
    service_level_1_embedding = tf.keras.layers.Embedding(len(X['service_level_1'].unique()), 3, input_length=1, name='service_level_1_embedding')(service_level_1)
    is_subdep_1_embedding = tf.keras.layers.Embedding(len(X['is_subdep_1'].unique()), 1, input_length=1, name='is_subdep_1_embedding')(is_subdep_1)
    is_csid_1_embedding = tf.keras.layers.Embedding(len(X['is_csid_1'].unique()), 1, input_length=1, name='is_csid_1_embedding')(is_csid_1)
    dayofweek_1_embedding = tf.keras.layers.Embedding(len(X['dayofweek_1'].unique()), 4, input_length=1, name='dayofweek_1_embedding')(dayofweek_1)
    day_part_1_embedding = tf.keras.layers.Embedding(len(X['day_part_1'].unique()), 2, input_length=1, name='day_part_1_embedding')(day_part_1)
    month_1_embedding = tf.keras.layers.Embedding(len(X['month_1'].unique()), 6, input_length=1, name='month_1_embedding')(month_1)
    week_1_embedding = tf.keras.layers.Embedding(len(X['week_1'].unique()), 26, input_length=1, name='week_1_embedding')(week_1)
    year_1_embedding = tf.keras.layers.Embedding(len(X['year_1'].unique()), 1, input_length=1, name='year_1_embedding')(year_1)
    person_1_embedding = tf.keras.layers.Embedding(len(X['person_1'].unique()), 1, input_length=1, name='person_1_embedding')(person_1)
    sole_1_embedding = tf.keras.layers.Embedding(len(X['sole_1'].unique()), 1, input_length=1, name='sole_1_embedding')(sole_1)
    legal_1_embedding = tf.keras.layers.Embedding(len(X['legal_1'].unique()), 1, input_length=1, name='legal_1_embedding')(legal_1)
    auto_ping_queue_1_embedding = tf.keras.layers.Embedding(len(X['auto_ping_queue_1'].unique()), 1, input_length=1, name='auto_ping_queue_1_embedding')(auto_ping_queue_1)
    requester_type_embedding = tf.keras.layers.Embedding(len(X['requester_type'].unique()), 2, input_length=1, name='requester_type_embedding')(requester_type)
    gender_embedding = tf.keras.layers.Embedding(len(X['gender'].unique()), 1, input_length=1, name='gender_embedding')(gender)

    concatenated = tf.keras.layers.Concatenate()([service_1_embedding, 
                                                  service_title_1_embedding,
                                                 mfc_1_embedding,
                                                 internal_status_1_embedding,
                                                 external_status_1_embedding,
                                                 order_type_1_embedding,
                                                 department_id_1_embedding,
                                                 custom_service_id_1_embedding,
                                                 service_level_1_embedding,
                                                 is_subdep_1_embedding,
                                                 is_csid_1_embedding,
                                                 dayofweek_1_embedding,
                                                 day_part_1_embedding,
                                                 month_1_embedding,
                                                 year_1_embedding,
                                                 person_1_embedding,
                                                 sole_1_embedding,
                                                 legal_1_embedding,
                                                 auto_ping_queue_1_embedding,
                                                 requester_type_embedding,
                                                 gender_embedding])
    #out = tf.keras.layers.Flatten()(concatenated)
    
    #out = tf.keras.layers.Dense(512, activation='relu')(out)
    #out = tf.keras.layers.Dense(256, activation='relu')(out)
    #out = tf.keras.layers.Dense(256, activation='relu')(out)
    #out = tf.keras.layers.Conv1D(128, 2, padding='same')(concatenated)
    #out = tf.keras.layers.LeakyReLU()(out)
    #out = tf.keras.layers.Conv1D(64, 2, padding='same')(out)
    #out = tf.keras.layers.Flatten()(out)
    conv_0 = tf.keras.layers.Conv1D(128, 3, padding='same', activation='relu')(concatenated)
    conv_1 = tf.keras.layers.Conv1D(128, 2, padding='same', activation='relu')(concatenated)
    conv_2 = tf.keras.layers.Conv1D(128, 6, padding='same', activation='relu')(concatenated)
    conv_0 = tf.keras.layers.Conv1D(64, 3, padding='same', activation='relu')(conv_0)
    conv_1 = tf.keras.layers.Conv1D(64, 2, padding='same', activation='relu')(conv_1)
    conv_2 = tf.keras.layers.Conv1D(64, 6, padding='same', activation='relu')(conv_2)
    concatenated_tensor = tf.keras.layers.Concatenate(axis=1)([conv_0, conv_1, conv_2])
    out = tf.keras.layers.Flatten()(concatenated_tensor)
    
    out = tf.keras.layers.Dense(len(y.unique()), activation='softmax', name='service_title')(out)

    model = tf.keras.models.Model(inputs=[service_1, 
                                          service_title_1, 
                                          mfc_1,
                                         internal_status_1,
                                         external_status_1,
                                         order_type_1,
                                         department_id_1,
                                         custom_service_id_1,
                                         service_level_1,
                                         is_subdep_1,
                                         is_csid_1,
                                         dayofweek_1,
                                         day_part_1,
                                         month_1,
                                         week_1,
                                         year_1,
                                         person_1,
                                         sole_1,
                                         legal_1,
                                         auto_ping_queue_1,
                                         requester_type,
                                         gender], 
                                  outputs=out)

    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)
    model.compile(loss = 'categorical_crossentropy', optimizer=optimizer, metrics=["accuracy"])

    return model


n_splits = 5

# +
VER='v5'
DISPLAY=1 # USE display=1 FOR INTERACTIVE

service_title = np.zeros((X.shape[0], 1))

skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777)


for fold, (idxT, idxV) in enumerate(skf.split(X,y)):

    print('#'*25)
    print('### FOLD %i'%(fold+1))
    print('#'*25)

    # Cleaning everything
    K.clear_session()
    tf.tpu.experimental.initialize_tpu_system(tpu)

    # Building model
    with strategy.scope():
        model = build_model()

    n_steps, trn_dataset, val_dataset = generate_dataset(idxT, idxV)

    reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)

    sv = tf.keras.callbacks.ModelCheckpoint(
        '%s-learnedemb-%i.h5'%(VER,fold), monitor='val_accuracy', verbose=1, save_best_only=True,
        save_weights_only=True, mode='auto', save_freq='epoch')

    hist = model.fit(trn_dataset,
                    epochs=EPOCHS, 
                    verbose=DISPLAY, 
                    callbacks=[sv, reduce_lr],
                    validation_data=val_dataset)

StarcoderdataJupyterScriptsDedupFiltered