# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/brucker3/KDM_spring2021/blob/main/icp4.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + colab={"base_uri": "https://localhost:8080/"} id="ATioFGVMwbSs" outputId="9843caba-bdaf-4d13-e362-3aece3afa8e1"
# downloading java for spark 
# !sudo apt-get update
# !apt-get install openjdk-8-jdk-headless -qq > /dev/null

# downloading  and set up spark 
# !wget -q https://www-us.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
# !tar xf spark-3.0.1-bin-hadoop2.7.tgz
# !pip install -q findspark

# + id="pe01c41YwkEf"
# set up env 
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-3.0.1-bin-hadoop2.7"

import findspark
findspark.init()

# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="Nggr_wEvxBzy" outputId="b6eb3b92-3ef7-4618-9ab5-ebc577ac8078"
# find spark 
findspark.find()


# + id="LGnBC0AQxDhN"
from pyspark.sql import SparkSession

spark = SparkSession.builder\
        .master("local")\
        .appName("icp4")\
        .config('spark.ui.port', '4050')\
        .getOrCreate()

# + colab={"base_uri": "https://localhost:8080/", "height": 216} id="_sB2zsVDyd7s" outputId="7972aab5-d44c-495a-c5c1-323cb0efdbc0"
#spark check that its set up correctly 
spark

# + colab={"base_uri": "https://localhost:8080/"} id="dwXwW0W-zF4p" outputId="ac841736-b6c2-439c-84a5-b0a8805f4dde"
# loading data set 

df = spark.read.csv("/content/data.csv", header=True, inferSchema= True)
df.printSchema()


# + colab={"base_uri": "https://localhost:8080/"} id="vtKNQDAC2nvp" outputId="46392aae-52be-4e7b-872f-89c6677f65af"
df.show(5, truncate=False)

# + [markdown] id="kukcgKtL5FW9"
# Creat new data set filter why its important 
# Transformation vs action 
# look at tuesday lecture for action vs transformation 

# + colab={"base_uri": "https://localhost:8080/"} id="S2UOj58q56yB" outputId="c8fe9948-eda9-4f43-bb8d-8eef1fe9c6e0"
# transforamiton  and action 1 
# so in total there are 7043  after filtering for tenure the count is 5073
df_1.count()
df_1 = df.filter(df.tenure > 10)
df_1.show()
df.count()

# + colab={"base_uri": "https://localhost:8080/"} id="w2EzkdC8-vEf" outputId="d9007b04-5586-47aa-d2d5-ee7fddefdf82"

from pyspark.sql.functions import sum
df_2 =df.select('SeniorCitizen','tenure', 'MonthlyCharges')
df_2.show(5)
print(df_2.groupBy().sum().collect())
# transformition and action 2 removed non quantative rows and summed them 
# some intering finding TotalCharges is not consired a quanttative row 
# so only three on 20 rows are 
# also of the 7043 only 1142 are senior for 16.2%

# + id="gQTiPNHmDIGl" colab={"base_uri": "https://localhost:8080/"} outputId="8a12c3dd-1b1b-428b-dc0e-761ee50e0e87"
# action and transfomriont 3 
# based on mean of monthly charges 
print(df.groupBy().mean().collect())

df_3 = df.filter(df.MonthlyCharges > 64)
df_3.show(5) # new df looks good 
# loook on the how the number changes once you take the hight side of the ave of monthly payments
# one inteertesing finding a majority of seniors pay more than the mean per month 
print(df_3.groupBy().sum().collect())
print(df_3.groupBy().mean().collect())



StarcoderdataJupyterScriptsDedupFiltered