import numpy as np
import pandas as pd
from datetime import datetime
from bosch_cat_features import categoric_fields, idList

chunksize = 50000

def pearson_for_features(fields_map, input_fname, output_fname1, output_fname2):
    i = 0
    for key in fields_map:
        print i, key, 
        field_list = idList + fields_map[key]
#        input_fname = output_fname1 + key + output_fname2
        t0 = datetime.now()
        use_header = True
        output_fname = output_fname1 + key + output_fname2
        for chunk in pd.read_csv(input_fname, usecols=field_list, iterator=True, chunksize=chunksize, dtype=object):
            print "*",
            data = pd.DataFrame()
            data = data.append(chunk, ignore_index=True)
            data.to_csv(output_fname, mode="a", header=use_header, index=False)
            use_header = False

        t1 = datetime.now()
        print input_fname, "loaded in", t1 - t0

        i += 1

def split_train_categoric():
    pearson_for_features(categoric_fields, "input/train_categorical.csv", "stations/train/", "_categoric.csv")

def split_test_categoric():
    pearson_for_features(categoric_fields, "input/test_categorical.csv", "stations/test/t", "_categoric.csv")

def process_cat_values(fields_map, input_fname, output_fname1, output_fname2):
    i = 0
    for key in fields_map:
        print i, key, 
        field_list = fields_map[key]
#        input_fname = output_fname1 + key + output_fname2
        if (len(field_list) > 0):
            t0 = datetime.now()
            output_fname = output_fname1 + key + output_fname2
            output_fname_new = output_fname1 + key + "_onehot" + output_fname2
            
            data = pd.read_csv(output_fname, usecols=field_list, dtype=object)
            print 'read...',
            data = pd.get_dummies(data)
            print 'converted...',
            data.to_csv(output_fname_new, index=False)
    
            t1 = datetime.now()
            print output_fname, "loaded in", t1 - t0
        else:
            print "0 columns, skipping..."
        i += 1
    

#split_train_categoric()
#split_test_categoric()

process_cat_values(categoric_fields, "", "stations/misc/", "_categoric.csv")
