# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd

# # Clean data

# ### Read files

# %ls

wifi_log_data = pd.read_csv("../cleaned_data/full.csv", names=["room", "event_time", "detec1", "detec2"])
wifi_log_data.head()

wifi_log_data.tail()

# ### Transform timestamps

# +
import time
from dateutil.parser import parse

for i in range(wifi_log_data.shape[0]):
    x = wifi_log_data["event_time"][i]
    y = parse(x)
    epoch = int(time.mktime(y.timetuple()))
    wifi_log_data.set_value(i,"event_time",epoch)
# -

wifi_log_data.head()

# ### Extract room data

for i in range(wifi_log_data.shape[0]):
    #the following line selects the last character of the string, which is the room ID
    wifi_log_data.set_value(i, "room", wifi_log_data["room"][i][-1:])

wifi_log_data.head()

wifi_log_data.tail()

# ### Add building

wifi_log_data["building"] = "school of computer science"

wifi_log_data.head()

occupancy_data = pd.read_csv("../cleaned_data/survey_data.csv")

occupancy_data.head()

occupancy_data.tail()

# ### Deleted unwated column

del occupancy_data["Unnamed: 0"]

occupancy_data.head()

# # Prepare data for analysis

# ### Transform EPOCH time into human-friendly time

occupancy_data['event_time'] = pd.to_datetime(occupancy_data.event_time, unit='s')
occupancy_data.head()

wifi_log_data['event_time'] = pd.to_datetime(wifi_log_data.event_time, unit='s')
wifi_log_data.set_index('event_time', inplace=True)
wifi_log_data.head()

# +
wifi_log_data['event_hour'] = wifi_log_data.index.hour
wifi_log_data['event_day'] = wifi_log_data.index.day

wifi_log_data.index.name = None

wifi_log_data.head()
# -

occupancy_data['event_time'] = pd.to_datetime(occupancy_data.event_time, unit='s')
occupancy_data.set_index('event_time', inplace=True)
occupancy_data.head()

# +
occupancy_data['event_hour'] = occupancy_data.index.hour
occupancy_data['event_day'] = occupancy_data.index.day

occupancy_data.index.name = None

occupancy_data.head()
# -

# ### Compute average number of wifi connections per room, day and hour
#
# ##### This is to make it easy to compare it with the survey data where we only have 1 recorded value per room, day and hour

df_mean_conn = wifi_log_data.groupby(['room', 'event_day', 'event_hour'], as_index=False).mean()

df_mean_conn.head()

# ### Merge the data into a single data frame

# +
df_mean_conn['room'] = df_mean_conn['room'].astype(int)

print(df_mean_conn.dtypes)
# -

occupancy_data.dtypes

# +
df = pd.merge(df_mean_conn, occupancy_data, on=['room', 'event_day', 'event_hour'], how='inner')

df.head(15)
# -

# ### add estimated occupancy
#
# ##### (number of occupants: number of seats in the room * occupancy % as indicated on the occupancy survey)

max_occupancy_b002_b003 = 90
max_occupancy_b004 = 220

# +
df['estim_nb_ocpnts'] = None

for i in range(df.shape[0]):
    if df['room'][i] == 2 or df['room'][i] == 3 :
        estim = df['occupancy'][i] * max_occupancy_b002_b003
        df.set_value(i,'estim_nb_ocpnts', estim)
    elif df['room'][i] == 4:
        estim = df['occupancy'][i] * max_occupancy_b004
        df.set_value(i,'estim_nb_ocpnts', estim)
    else:
        raise ValueError("Incorrect room number:", df['room'][i])
# -

df.head(15)

df.tail(15)

# ### save to csv

df.to_csv('wifi_logs_occupancy_survey_merged_df.csv')

# # Data analytics

# ### Add dummies

# +
dummies = pd.get_dummies(df['occupancy']).rename(columns=lambda x: 'Category_' + str(x).replace(".", ""))
df = pd.concat([df, dummies], axis=1)

df.head()
# -

df[['detec2', 
            'Category_00', 
            'Category_025', 
            'Category_05', 
            'Category_075',
            'Category_10']].corr()

# +
import matplotlib.pyplot as plt
# %matplotlib inline

fig, axs = plt.subplots(1, 5, sharey=True)

df.plot(kind='scatter', x='detec2', y='Category_00', label="%.3f" 
        % df[['detec2', 'Category_00']].corr().as_matrix()[0,1], ax=axs[0], figsize=(15, 8))

df.plot(kind='scatter', x='detec2', y='Category_025', label="%.3f" 
        % df[['detec2', 'Category_025']].corr().as_matrix()[0,1], ax=axs[1])

df.plot(kind='scatter', x='detec2', y='Category_05', label="%.3f" 
        % df[['detec2', 'Category_05']].corr().as_matrix()[0,1], ax=axs[2])

df.plot(kind='scatter', x='detec2', y='Category_075', label="%.3f" 
        % df[['detec2', 'Category_075']].corr().as_matrix()[0,1], ax=axs[3])

df.plot(kind='scatter', x='detec2', y='Category_10', label="%.3f"
        % df[['detec2', 'Category_10']].corr().as_matrix()[0,1], ax=axs[4])
# -

from patsy import dmatrices
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import train_test_split
from sklearn import metrics
from sklearn.cross_validation import cross_val_score


# +
def calculate_logreg_details(df, feature, dummy):
    '''provides details of the calculation of the logistic regression between 1 feature and 1 target.
    This function does not return.
    
    Parameters
    ----------
        df: a pandas dataframe
        feature: predictive feature
        dummy: target feature (binary)
    
    Returns
    ---------
        void
    '''
    intercept = pd.DataFrame({'Intercept': np.ones(df.shape[0])})
    X = pd.concat([intercept, df[[feature, dummy]]], axis=1)
    y = df[dummy]
    print("Descriptive features:\n", X)
    print()
    print("\nTarget feature:\n", y)
    print()
    logreg = LogisticRegression().fit(X[['Intercept', feature]], y)
    print("Coeficients: \n", logreg.coef_)
    print()
    logreg.predict_proba(X[['Intercept', feature]])
    predictions = logreg.predict(X[['Intercept', feature]])
    print("Predictions: ", predictions)
    print()
    print("Accuracy:", logreg.score(X[['Intercept', feature]], y))

    
def calculate_logreg(df, feature, dummy):
    '''provide the logistic regression accuracy between a feature and a target.
    
    Parameters
    ----------
        df: a pandas dataframe
        feature: predictive feature
        dummy: target feature (binary)
    
    Returns
    ---------
        float: accuracy of the logistic regression
    '''
    
    intercept = pd.DataFrame({'Intercept': np.ones(df.shape[0])})
    X = pd.concat([intercept, df[[feature, dummy]]], axis=1)
    y = df[dummy]
    logreg = LogisticRegression().fit(X[['Intercept', feature]], y)
    logreg.predict_proba(X[['Intercept', feature]])
    predictions = logreg.predict(X[['Intercept', feature]])
    return logreg.score(X[['Intercept', feature]], y)


# -

calculate_logreg_details(df, 'detec2', 'Category_00')

print("Accuracy: predicting occupancy 0%:", calculate_logreg(df, 'detec2', 'Category_00'))
print("Accuracy: predicting occupancy 25%:", calculate_logreg(df, 'detec2', 'Category_025'))
print("Accuracy: predicting occupancy 50%:", calculate_logreg(df, 'detec2', 'Category_05'))
print("Accuracy: predicting occupancy 75%:", calculate_logreg(df, 'detec2', 'Category_075'))
print("Accuracy: predicting occupancy 100%:", calculate_logreg(df, 'detec2', 'Category_10'))

calculate_logreg_details(df, 'detec2', 'Category_025')

calculate_logreg_details(df, 'detec2', 'Category_10')

df.head(20)

df.tail(20)

df.occupancy.value_counts()

df.loc[df['occupancy'] == 1.0]

help(pd)



StarcoderdataJupyterScriptsDedupFiltered