# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import gc
import os
import pandas as pd
from glob import glob
from itertools import combinations

sorted(glob(f"../data/interim/features/*/features*.hdf.compress"))

sorted(glob(f"../data/interim/train_2017-11-0*00.csv"))

dtypes = {
    'ip': 'uint32',
    'app': 'uint16',
    'device': 'uint16',
    'os': 'uint16',
    'channel': 'uint16',
    'is_attributed': 'uint8'
}
to_read = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']

# +
target_entities_init = ['app', 'device', 'os', 'channel', 'ip']
target_entities = []

for t in combinations(target_entities_init, 1):
    target_entities.append(t[0])
for t in combinations(target_entities_init, 2):
    target_entities.append(list(t))
# for t in combinations(target_entities_init, 3):
#     target_entities.append(list(t))
# for t in combinations(target_entities_init, 4):
#     target_entities.append(list(t))

target_entities.remove(['channel', 'ip'])
# -

train_files = [
    '../data/interim/train_2017-11-07_1700.csv',
    '../data/interim/train_2017-11-07_1800.csv',
    '../data/interim/train_2017-11-07_1900.csv',
    '../data/interim/train_2017-11-07_2000.csv',
    '../data/interim/train_2017-11-07_2100.csv',
    '../data/interim/train_2017-11-07_2200.csv',
    '../data/interim/train_2017-11-07_2300.csv',
    '../data/interim/train_2017-11-08_0000.csv',
    '../data/interim/train_2017-11-08_0100.csv',
    '../data/interim/train_2017-11-08_0200.csv',
    '../data/interim/train_2017-11-08_0300.csv',
    '../data/interim/train_2017-11-08_0400.csv',
    '../data/interim/train_2017-11-08_0500.csv',
    '../data/interim/train_2017-11-08_0600.csv',
    '../data/interim/train_2017-11-08_0700.csv',
    '../data/interim/train_2017-11-08_0800.csv',
    '../data/interim/train_2017-11-08_0900.csv',
    '../data/interim/train_2017-11-08_1000.csv',
    '../data/interim/train_2017-11-08_1100.csv',
    '../data/interim/train_2017-11-08_1200.csv',
    '../data/interim/train_2017-11-08_1300.csv',
    '../data/interim/train_2017-11-08_1400.csv',
    '../data/interim/train_2017-11-08_1500.csv',
    '../data/interim/train_2017-11-08_1600.csv'
]

validation_files = [
    '../data/interim/train_2017-11-09_0400.csv',
    '../data/interim/train_2017-11-09_0500.csv',
    '../data/interim/train_2017-11-09_0600.csv',
]


def split(df, target_entity):
    if type(target_entity) == str:
        df[target_entity] = df.index
    else:
        df[target_entity[0]], df[target_entity[1]] = df.index.str.split('_', 1).str
        df[target_entity[0]] = df[target_entity[0]].astype(dtypes[target_entity[0]])
        df[target_entity[1]] = df[target_entity[1]].astype(dtypes[target_entity[1]])
    return df


def combine_features(train_files, features_prefix, feature_suffix):
    for train_file in train_files:
        df_train = pd.read_csv(train_file, usecols=to_read, dtype=dtypes)
        name, ext = os.path.splitext(os.path.basename(train_file))
        for target_entity in target_entities:
            target_entity_name = target_entity if type(target_entity) == str else "_".join(target_entity)
            feature_files = sorted(glob(f"../data/interim/features/{target_entity_name}/{features_prefix}*{feature_suffix}.hdf.compress"))
            assert len(feature_files) > 0
            for feature_file in feature_files:
                df_feature = pd.read_hdf(feature_file)
                df_feature = split(df_feature, target_entity)
                df_train = pd.merge(df_train, df_feature, how='left', left_on=target_entity, right_on=target_entity)
                del df_feature
                gc.collect()

        filename_output = f"../data/interim/combined/{name}_{feature_suffix}.hdf.compress"
        print(f"Writing {filename_output}")
        df_train.to_hdf(filename_output, 'train', mode='w', complib='blosc', fletcher32=True, complevel=9)
        del df_train
        gc.collect()


combine_features(train_files, 'features_2017-11-07_1700', 'attributed2')

combine_features(validation_files, 'features_2017-11-08_1700', 'attributed2')

# %%time
df = pd.read_hdf('../data/interim/combined/train_2017-11-07_1700_attributed2.hdf.compress')

sorted(glob(f"../data/interim/combined/train_2017-11-0*.hdf.compress"))

train_feature_files = ['../data/interim/combined/train_2017-11-07_1700_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_1800_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_1900_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_2000_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_2100_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_2200_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-07_2300_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0000_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0100_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0200_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0300_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0400_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0500_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0600_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0700_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0800_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_0900_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1000_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1100_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1200_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1300_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1400_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1500_attributed.hdf.compress',
 '../data/interim/combined/train_2017-11-08_1600_attributed.hdf.compress']

df_train = pd.concat([pd.read_hdf(f) for f in train_feature_files], ignore_index=True)

df_train.columns

df_train.to_hdf('../data/processed/train_2017-11-07_1700_08_1600_attributed.hdf.compress', 'train', mode='w', complib='blosc', fletcher32=True)



StarcoderdataJupyterScriptsDedupFiltered