# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/ilkert08/pyspark/blob/main/pyspark.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="TDY9LsSMvjIS"
# !apt-get install openjdk-8-jdk-headless -qq > /dev/null
# !wget -q http://apache.osuosl.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
# !tar xf spark-3.1.2-bin-hadoop3.2.tgz
# !pip install -q findspark

# + id="eYShR3rgvtLg"
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-3.1.2-bin-hadoop3.2"

# + id="hiXKk7ncwMFs"
import pandas as pd
import numpy as np
import findspark
findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").getOrCreate()

# + colab={"base_uri": "https://localhost:8080/", "height": 203} id="eH35n1ZpwMJE" outputId="69df4dca-cec7-4159-f29a-150bc6700f7d"
from sklearn.datasets import load_iris
data= load_iris()
df = pd.DataFrame(data.data, columns=['sepalLength', 'sepalWidth', 'petalLength', 'petalWidth'])
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="JP1c9NDIwMMx" outputId="5e036ea1-7c6a-4445-e3f7-6eb5302e5484"
spark = SparkSession.builder.appName("Python Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
sparkDF=spark.createDataFrame(df) 
sparkDF.printSchema()
sparkDF.show()



# + id="WCzQt7l16jDF"


# + colab={"base_uri": "https://localhost:8080/"} id="PK2k6utzwMP3" outputId="a2da5eba-9681-4501-a928-b90ad473100e"
sparkDF.createOrReplaceTempView("iris")
query = "select * from iris where sepalLength > 5;"
sqlDF = spark.sql(query)
sqlDF.show()

# + colab={"base_uri": "https://localhost:8080/"} id="5A6iYqx5wMSz" outputId="bd5c11a9-ce23-4bba-b342-e585402e1006"
data2 = sparkDF.groupBy('sepalLength').count().orderBy('count', ascending = True)
data2.collect()

StarcoderdataJupyterScriptsDedupFiltered