# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import findspark
findspark.init()

#spark connection and import all required packages 
from pyspark.sql import SparkSession,types

spark = SparkSession.builder.master("local").appName("Create DataFrame").getOrCreate()

sc = spark.sparkContext

from datetime import datetime, date
rdd = spark.sparkContext.parallelize([
    (1, 2., 'su1', date(2001, 4, 1), datetime(2001, 4, 1, 12, 0)),
    (2, 3., 'su3', date(2002, 6, 2), datetime(2002, 6, 2, 12, 0)),
    (3, 4., 'su5', date(2003, 3, 3), datetime(2003, 3, 3, 12, 0))
])

df = spark.createDataFrame(rdd, schema=['a', 'b', 'c', 'd', 'e'])
df

df.show()

StarcoderdataJupyterScriptsDedupFiltered