# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 
import os
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter
%matplotlib inline
print(os.listdir("../input"))
"""
# Data Manipulate 
"""
data=pd.read_csv('../input/complete_dataset.csv')
data.head()
print(data.shape)
print(data.columns)
data.info()
print(data["testResultMetricValue"].isnull().values.any())
#print(data["testResultMetricValue"].value_counts().sort_values( ascending=False))
#data.isnull().sum()
data['participantCreatedOn'] = pd.to_datetime(data['participantCreatedOn'], errors='coerce')
data['testResultMetricCreatedOn'] = pd.to_datetime(data['testResultMetricCreatedOn'], errors='coerce')
data['measurementDate'] = data['testResultMetricCreatedOn'].dt.strftime('%B %d, %Y')
data['full_test_name'] = data[['testName', 'testMetricName']].apply(lambda x: '-'.join(x), axis=1)
data.drop(["Unnamed: 19",'testResultMetricCreatedOn'],axis=1,inplace=True)
print(data.shape, data.columns)
display(data.head(3))
for col in [ 'participantWeightLbs', 'participantHeightCms',
               'participantBirthYear','testResultMetricValue']:
    data[col] = pd.to_numeric(data[col],errors='coerce')

data['participantIsControl'].value_counts().plot(kind='bar')
plt.legend( ( 'MS','Healty'))
#data['participantIsControl'] = data['participantIsControl'].astype(int)
data.describe().T
ds_overview_df2 = data.groupby('floodlightOpenId').\
  size().reset_index(name='count').sort_values('count', ascending=False)
print(ds_overview_df2.shape)
ds_overview_df2.head(5)
data.groupby("participantSex").size().plot(kind='bar')#data.groupby("participantSex")['participantIsControl'].size() filter ile dene
plt.legend( ( 'Female','Male'))
"""
data.groupby('participantIsControl') ['participantBirthYear'].value_counts()
data['floodlightOpenId'].value_counts().sort_values(ascending=False)
data["testResultMetricValue"].describe()

"""

part_cols=['floodlightOpenId', 'measurementDate',*list(data)[1:8]]
raw_measurement_df=data.pivot_table(index=part_cols,columns=['full_test_name'],values=['testResultMetricValue'])
raw_measurement_df.head()
print(raw_measurement_df.shape)
raw_measurement_df.info()
measurement_df = raw_measurement_df.reset_index()
measurement_df.to_csv('clean_measure_table.csv', index=False)
"""
print(measurement_df.floodlightOpenId.value_counts())
raw_measurement_df.index.names
measurement_df.info()
"""
#d=measurement_df.groupby('participantCountryOfResidence')['participantIsControl'].value_counts().to_frame()
#d.head(5)
measurement_df.groupby("participantSex")["participantIsControl"].value_counts().unstack().plot(kind='bar')
plt.legend( ( 'MS','Healty'))#1:healty 0:MS
plt.xticks(rotation= 45)
#list(measurement_df["participantCountryOfResidence"].unique())
plt.figure(figsize=(100,100))
measurement_df.groupby(['participantCountryOfResidence','participantIsControl']).size().unstack().plot(kind='bar')
plt.legend( ( 'MS','Healty'))#1:healty 0:MS
plt.xticks(rotation= 45)
plt.show()
plt.figure(figsize=(10,10))
measurement_df.groupby(['participantBirthYear','participantIsControl']).size().unstack().plot(kind='bar',stacked=True)
plt.show()
measurement_df.isnull().sum()#NaN values count
#measurement_df.apply(lambda x: x.fillna(x.mean(),axis=0))
"""
### Missing value graph
"""
measurement_df[measurement_df.notnull().all(axis=1)].count()#hiçbir nan değer olmayan satır sayısı
import missingno as msno
msno.bar(measurement_df)
import seaborn as sns
sns.heatmap(measurement_df.isnull(),cbar=False)
msno.heatmap(measurement_df)#nullity korelasyon heatmapi
#1 yüksek korelasyon eksikliklerin biribiri ile ilişkisi var
"""
### Missing Value
"""
from sklearn.impute import SimpleImputer
df=measurement_df.copy()
var_name=list(df)
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
imp_mean=imp_mean.fit(df[var_name[9:]])
df[var_name[9:]]=imp_mean.transform(df[var_name[9:]])
df.head(5)
"""
from sklearn.preprocessing import RobustScaler
col=list(df.select_dtypes(include=[np.number]).columns.values)
transformer = RobustScaler().fit(df[col])
df[col]=transformer.transform(df[col])
df.head()
"""
"""
### Normalizasyon
"""
from sklearn.preprocessing import StandardScaler
col=list(df.select_dtypes(include=[np.number]).columns.values)
transformer = StandardScaler(with_mean=False).fit(df[col])
df[col]=transformer.transform(df[col])
df.head()

"""
c=list(measurement_df.columns[9:].values)
measurement_df[c[1]].fillna(measurement_df.groupby("participantSex")[c[1]].transform("mean"))
measurement_df.groupby("participantSex")['testResultMetricValue', 'Five UTurn Test-Turn Speed Average'].count()
   
    

"""
"""
# Logistic Regression 
"""
"""
df['participantIsControl']=df['participantIsControl'].astype(int)
y=df['participantIsControl']
col=list(df.select_dtypes(include=[np.number]).columns.values)
del col[0:2] 
x=df[col]
loj=LogisticRegression(solver="newton-cg")
loj_model=loj.fit(x,y)
y_pred=loj_model.predict(x)
accuracy_score(y,y_pred)
"""
from sklearn import preprocessing
lenc = preprocessing.LabelEncoder()
part_cols = [ "participantBirthYear", "participantWeightLbs", "participantHeightCms"]
df[part_cols] = df[part_cols].apply(lenc.fit_transform)
df.head()
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
x_vars = raw_measurement_df.columns
x=df[col]
y=df['participantIsControl'].astype(int)
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
logreg = LogisticRegression(solver="newton-cg")
logreg.fit(X_train, y_train)
y_pred = logreg.predict(X_test)
print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(X_test, y_test)))
from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_test,y_pred)
print(confusion_matrix)
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc = roc_auc_score(y_train, logreg.predict(X_train))
fpr, tpr, thresholds = roc_curve(y_train, logreg.predict_proba(X_train)[:,1])
plt.figure()
plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.savefig('Log_ROC')
plt.show()