# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

dataset = spark.read.csv("file:///home/guri/Desktop/Wine-Dataset-Classification-Using-Spark-Mlib/wine.csv",header=True,inferSchema=True)

from pyspark.ml.classification import LogisticRegression

dataset.printSchema()

dataset.show(5)

# +
from pyspark.sql.functions import isnan, when, count, col

dataset.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in dataset.columns]).show()
# -

dataset.columns

# +
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
 
assembler=VectorAssembler(inputCols=['Alcohol',
 'Malic acid',
 'Ash',
 'Alcalinity of ash',
 'Magnesium',
 'Total phenols',
 'Flavanoids',
 'Nonflavanoid phenols',
 'Proanthocyanins',
 'Color intensity',
 'Hue',
 'OD280/OD315 of diluted wines',
 'Proline'],outputCol='features')
 
output_data=assembler.transform(dataset)
# -

final_data=output_data.select('features','Class')    
train,test=final_data.randomSplit([0.7,0.3])
model=LogisticRegression(labelCol='Class')           
model=model.fit(train)        
summary=model.summary 
summary.predictions.describe().show()

# +
from pyspark.ml.evaluation import BinaryClassificationEvaluator

predictions=model.evaluate(test)


StarcoderdataJupyterScriptsDedupFiltered