# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: conda_python3
#     language: python
#     name: conda_python3
# ---

# +
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('tree').getOrCreate()
df = spark.read.csv('College.csv', inferSchema=True, header=True)
df.printSchema()

# +
import pandas as pd

pd.DataFrame(df.take(5), columns = df.columns).transpose()
# -

df.columns

# +
from pyspark.ml.feature import VectorAssembler

assembler = VectorAssembler(inputCols = ['Apps','Accept','Enroll','Top10perc','Top25perc','F_Undergrad','P_Undergrad','Outstate','Room_Board','Books','Personal','PhD','Terminal','S_F_Ratio','perc_alumni','Expend','Grad_Rate'], outputCol = 'features')
# -

output = assembler.transform(df)

# +
from pyspark.ml.feature import StringIndexer

indexer = StringIndexer(inputCol = 'Private', outputCol = 'PrivateIndex')
outputFixed = indexer.fit(output).transform(output)
outputFixed.printSchema()
# -

final_df = outputFixed.select('features', 'PrivateIndex')
final_df.show(3)

train, test = final_df.randomSplit([0.7, 0.3])

# +
from pyspark.ml.classification import (DecisionTreeClassifier, RandomForestClassifier, 
                                      GBTClassifier)
from pyspark.ml import Pipeline

dt = DecisionTreeClassifier(labelCol = 'PrivateIndex', featuresCol = 'features')
rf = RandomForestClassifier(labelCol = 'PrivateIndex', featuresCol = 'features')
gb = GBTClassifier(labelCol = 'PrivateIndex', featuresCol = 'features')
# -

dt_model = dt.fit(train)
rf_model = rf.fit(train)
gb_model = gb.fit(train)

dt_predictions = dt_model.transform(test)
rf_predictions = rf_model.transform(test)
gb_predictions = gb_model.transform(test)

# +
from pyspark.ml.evaluation import BinaryClassificationEvaluator

binary_evaluator = BinaryClassificationEvaluator(labelCol = 'PrivateIndex')

print('Decision Tree:', binary_evaluator.evaluate(dt_predictions))
# -

print('Random Forest:' , binary_evaluator.evaluate(rf_predictions))

print('Gradient-boosted Trees:', binary_evaluator.evaluate(gb_predictions))

# +
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

multi_evaluator = MulticlassClassificationEvaluator(labelCol = 'PrivateIndex', metricName = 'accuracy')
print('Decision Tree Accu:', multi_evaluator.evaluate(dt_predictions))
# -

multi_evaluator.getMetricName()

print('Random Forest Accu:', multi_evaluator.evaluate(rf_predictions))

print('Gradient-boosted Trees Accu:', multi_evaluator.evaluate(gb_predictions))

StarcoderdataJupyterScriptsDedupFiltered