from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession

sc = SparkContext(conf=SparkConf())
spark = SparkSession(sparkContext=sc)

"""
Explanation: Binary Classification
This notebook follows exactly the tutorial from Kaggle: data analysis framework from Kaggle. Thanks to the author LD Freeman for creating such a great tutorial! My goal is to create an Apache Spark version using the same framework.
Create Spark entry points
End of explanation
"""


# load packages
import sys
print('Python version: {}'. format(sys.version))

import pandas as pd
print('Python version: {}'. format(pd.__version__))

import matplotlib
print('matplotlib version: {}'. format(matplotlib.__version__))

import numpy as np
print('numpy version: {}'. format(np.__version__))

import scipy as sp
print('scipy version: {}'. format(sp.__version__))

import IPython
from IPython import display # pretty printing of dataframe in Jupyter notebook
print('IPython version: {}'. format(IPython.__version__))

import pyspark
print('Apache Spark Pyspark version: {}'. format(pyspark.__version__)) # pyspark version

# misc libraries
import random
import time

# ignore warnings
import warnings
warnings.filterwarnings('ignore')
print('-'*25)

"""
Explanation: Step 1: define the problem
 What sorts of people were likely to survive from the Titanic accident?
Step 2: gather the data
The datasets can be found here: https://www.kaggle.com/c/titanic/data.
It is also available in this github repository:

data/titanic/gender_submission.csv
data/titanic/test.csv
data/titanic/train.csv

Step 3: prepare data for consumption
3.1 Import libraries
3.11 Import python packages
End of explanation
"""


from pyspark.ml.classification import LinearSVC
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.classification import GBTClassifier
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.classification import NaiveBayes
from pyspark.ml.classification import MultilayerPerceptronClassifier
from pyspark.ml.classification import OneVsRest

"""
Explanation: 3.12 Import Pyspark Models for binary classification
End of explanation
"""


from subprocess import check_output
print('-'*10, 'datasets', '-'*10)
print(check_output(['ls', 'data/titanic']).decode('utf8'))

# import data
# we will split the train data into train and test data in future sections
data_raw = spark.read.csv('data/titanic/train.csv', inferSchema=True, header=True)

# the test file provided is for validation of final model.
data_val = spark.read.csv('data/titanic/test.csv', inferSchema=True, header=True)

# preview the data
# data type
print('-'*10, 'data types', '-'*10)
pd.DataFrame(data_raw.dtypes)

# data summary
print('-'*10, 'data summary', '-'*10)
data_raw.describe().toPandas()

# view a small subset of the data
print('-'*10, 'randomely sample 1% data to view', '-'*10)
data_raw.randomSplit([0.01, 0.99])[0].toPandas()

"""
Explanation: 3.2 Meet and greet data
End of explanation
"""


# we first check which values are NULL values for each column
# then we convert the boolean values to int (0 and 1), then we can count how many 1's exist in each column.
print('-'*25)
print('0: is not NULL')
print('1: is NULL')
print('-'*25)
print(' '*25)
# we build column strings and then use eval() to convert strings to column expressions.
data_raw.select([eval('data_raw.' + x + '.isNull().cast("int").alias("' + x + '")') for x in data_raw.columns]).show(n=10)

print('Train columns with null values:')
print('-'*25)
data_raw.select([eval('data_raw.' + x + '.isNull().cast("int").alias("' + x + '")') for x in data_raw.columns]).\
    groupBy().sum().toPandas()

print('Test columns with null values:')
print('-'*25)
data_val.select([eval('data_val.' + x + '.isNull().cast("int").alias("' + x + '")') for x in data_val.columns]).\
    groupBy().sum().toPandas()

"""
Explanation: 3.21 The 4 C's of data clearning: Correcting, Completing, Creating, and Converting
End of explanation
"""


# COMPLETE: complete or delete missing values in train and test/validation dataset.

# complete missing age with median

# complete missing embarked with mode

# complete missing fare with median

data_raw.select('Age')

"""
Explanation: 3.22 Clean data
COMPLETE
End of explanation
"""


