// ---
// jupyter:
//   jupytext:
//     text_representation:
//       extension: .scala
//       format_name: light
//       format_version: '1.5'
//       jupytext_version: 1.14.4
//   kernelspec:
//     display_name: spylon-kernel
//     language: scala
//     name: spylon-kernel
// ---

spark

// + language="python"
// from pyspark.ml.linalg import Vectors
// import numpy as np

// + language="python"
// # Load MNIST
// import tensorflow as tf
// mnist = tf.keras.datasets.mnist
//
// (x_train, y_train),(x_test, y_test) = mnist.load_data()
// x_train, x_test = x_train / 255.0, x_test / 255.0
// #print(x_train.shape, x_test.shape)
// x_train=x_train.flatten().reshape(60000, 28*28)
// x_test=x_test.flatten().reshape(10000, 28*28)
// y_train=y_train.flatten().reshape(60000, 1)
// y_test=y_test.flatten().reshape(10000, 1)

// + language="python"
// x_train.shape

// + language="python"
// y_x_tr = np.hstack([y_train, x_train])
// dff = map(lambda y_x: (
//                     int(y_x[0]), Vectors.dense(y_x[1:])
//                     ),  y_x_tr    
//          )
//
// mnistdf = spark.createDataFrame(dff, schema=["label", "features"]).cache()
// #mnistdf = spark.createDataFrame(, schema=["label", "features"])

// + language="python"
// mnistdf.take(1)

// + language="python"
// print(x_train.shape)
// print(y_train.shape)
// x_y_tr = np.concatenate(x_train, y_train)
// -



// + language="python"
// from sklearn.linear_model import LogisticRegression
// clf = LogisticRegression()
// clf.fit(x_train, y_train)

// + language="python"
// from sklearn import metrics
// from sklearn.preprocessing import LabelBinarizer
// enc = LabelBinarizer()
// enc.fit(y_test)
// pred = clf.predict_proba(x_test)
// pred_cond = enc.inverse_transform(pred)
//
// #y_te_exp = enc.transform(y_test)
//
// print(pred[:1])
// print(metrics.accuracy_score(y_test, pred_cond))

// + language="python"
// from sklearn.
// -



StarcoderdataJupyterScriptsDedupFiltered