# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.7.4 64-bit (''conda-env'': conda)'
#     language: python
#     name: python37464bitcondaenvconda3f4d7783b740440a9c80fd58cd6c0e7e
# ---

from pyspark.sql import SparkSession

spark = SparkSession \
        .builder \
        .appName("reading csv") \
        .getOrCreate()

# ### SparkSession.builder will return a builder.

type(SparkSession.builder)

type(SparkSession.builder.\
     appName("reading csv"))

# ### SparkSession.Builder can accept
#
# - SparkConf
# - SparkSession‘s own configuration

from pyspark.conf import SparkConf
type(SparkSession.builder.\
     appName("reading csv").\
     config(conf=SparkConf()))

type(SparkSession.builder.\
     config("spark.some.config.option", "some-value"))

type(SparkSession.builder.\
     appName("reading csv").\
     config(conf=SparkConf()).\
     enableHiveSupport())

# ### Call GetOrCreate to return the SparkSession.

type(SparkSession.builder.\
     appName("reading csv").\
     config(conf=SparkConf()).\
     enableHiveSupport().\
     getOrCreate())

# ## References
#
#   - <https://spark.apache.org/docs/2.1.0/api/python/pyspark.sql.html>

StarcoderdataJupyterScriptsDedupFiltered