# create entry points to spark
try:
    sc.stop()
except:
    pass
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
sc=SparkContext()
spark = SparkSession(sparkContext=sc)

"""
Explanation: DataFrame object

Create SparkContext and SparkSession
End of explanation
"""


mtcars = spark.read.csv(path='../../data/mtcars.csv',
                        sep=',',
                        encoding='UTF-8',
                        comment=None,
                        header=True, 
                        inferSchema=True)
mtcars.show(n=5, truncate=False)

"""
Explanation: Create a DataFrame object
Creat DataFrame by reading a file
End of explanation
"""


from pyspark.sql import Row
rdd = sc.parallelize([
    Row(x=[1,2,3], y=['a','b','c']),
    Row(x=[4,5,6], y=['e','f','g'])
])
rdd.collect()

df = spark.createDataFrame(rdd)
df.show()

"""
Explanation: Create DataFrame with createDataFrame function
From an RDD
Elements in RDD has to be an Row object
End of explanation
"""


import pandas as pd
pdf = pd.DataFrame({
    'x': [[1,2,3], [4,5,6]],
    'y': [['a','b','c'], ['e','f','g']]
})
pdf

df = spark.createDataFrame(pdf)
df.show()

"""
Explanation: From pandas DataFrame
End of explanation
"""


my_list = [['a', 1], ['b', 2]]
df = spark.createDataFrame(my_list, ['letter', 'number'])
df.show()

df.dtypes

my_list = [['a', 1], ['b', 2]]
df = spark.createDataFrame(my_list, ['my_column'])
df.show()

df.dtypes

"""
Explanation: From a list
Each element in the list becomes an Row in the DataFrame.
End of explanation
"""


my_list = [(['a', 1], ['b', 2])]
df = spark.createDataFrame(my_list, ['x', 'y'])
df.show()

"""
Explanation: The following code generates a DataFrame consisting of two columns, each column is a vector column.
Why vector columns are generated in this case?
In this case, the list my_list has only one element, a tuple. Therefore, the DataFrame has only one row. This tuple has two elements. Therefore, it generates a two-columns DataFrame. Each element in the tuple is a list, so the resulting columns are vector columns.
End of explanation
"""


