# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

#Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
import os
import re
# %matplotlib inline

def load_production_data(data_path="./"):
    
    production_train = []
    train_files = sorted(os.listdir(data_path+"/TrainingDataProduction/TrainingDataProduction"))
    train_files.sort(key=lambda f: int(re.sub('\D', '', f)))
    for file in train_files:
        temp = pd.read_csv(data_path+"/TrainingDataProduction/TrainingDataProduction/"+file, sep=";")
        temp["ProcessCycle"] = int(file.split('.')[0])
        production_train.append(temp)

    production_train = pd.concat(production_train)

    production_valid = []
    valid_files = sorted(os.listdir(data_path+"/ValidationDataProduction/ValidationDataProduction"))
    valid_files.sort(key=lambda f: int(re.sub('\D', '', f)))
    for file in valid_files:
        temp = pd.read_csv(data_path+"/ValidationDataProduction/ValidationDataProduction/"+file, sep=";")
        temp["ProcessCycle"] = int(file.split('.')[0])
        production_valid.append(temp)

    production_valid = pd.concat(production_valid)
    
    production_test = []

    test_files = sorted(os.listdir(data_path+"/EvaluationDataProduction/EvaluationDataProduction"))
    test_files.sort(key=lambda f: int(re.sub('\D', '', f)))
    for file in test_files:
        temp = pd.read_csv(data_path+"/EvaluationDataProduction/EvaluationDataProduction/"+file, sep=";")
        temp["ProcessCycle"] = int(file.split('.')[0])
        production_test.append(temp)

    production_test = pd.concat(production_test)
    
    cols_with_no_variance_train = []
    cols_with_no_variance_valid = []

    for col in production_train.columns:
        if len(production_train[col].unique()) == 1:
            cols_with_no_variance_train.append(col)

    for col in production_valid.columns:
        if len(production_valid[col].unique()) == 1:
            cols_with_no_variance_valid.append(col)

    common_cols_with_no_variance = list(set(cols_with_no_variance_train) & set(cols_with_no_variance_valid))

    production_train = production_train.drop(common_cols_with_no_variance, axis = 1)
    production_valid = production_valid.drop(common_cols_with_no_variance, axis = 1)
    production_test = production_test.drop(common_cols_with_no_variance, axis = 1)

    production_train["Split"] = "Train"
    production_valid["Split"] = "Valid"
    production_test["Split"] = "Test"
    
    
    production_train_val_combined = pd.concat([production_train, production_valid]).reset_index(drop=True)
    
    return production_train_val_combined, production_test

# +
df_train, df_test = load_production_data()


# -

train, valid = df_train, df_test

# +
#Erstellen von Dummies, um numerische Werte zu erhalten
split = pd.get_dummies(train['Split'],drop_first=True)

train.drop(["Split"],axis=1,inplace=True)
train = pd.concat([train,split],axis=1)
# -

train.info()

from sklearn.model_selection import train_test_split

# +
#Init von Trainings- und Testdaten
X_train, X_test, y_train, y_test = train_test_split(train.drop('Valid',axis=1), 
                                                    train['Valid'], test_size=0.30, 
                                                    random_state=101)

y_train = y_train.values
y_test = y_test.values

# +
from imblearn.over_sampling import SMOTE

sm = SMOTE(random_state=12, ratio = 1.0)
X_train, y_train = sm.fit_sample(X_train, y_train)

print(X_train.shape)
print(X_test.shape)
print(len(y_train[y_train==0]), 'negative samples in the training set')
print(len(y_train[y_train==1]), 'positive samples in the training set')
print(len(y_test[y_test==0]), 'negative samples in the test set')
print(len(y_test[y_test==1]), 'positive samples in the test set')
# -

#Principal Component Analysis für die Dimensionsreduktion
from sklearn.decomposition import PCA
# Make an instance of the Model
pca = PCA(n_components = 2)
pca_2d = pca.fit_transform(X_train)

X_train.info()

plt.figure(figsize = (10, 10))
plt.scatter(pca_2d[:,0], pca_2d[:,1], c = y_train)
plt.title('PCA scatter plot')
plt.show()

from sklearn.linear_model import LogisticRegression, LogisticRegressionCV, SGDClassifier
from sklearn.model_selection import GridSearchCV

# +
#Erstellen und Fitten des Modells auf Trainingsdaten
logmodel = LogisticRegression(solver="liblinear", max_iter=250)
#logmodel = LogisticRegressionCV(cv= 5, class_weight="balanced")
logmodel.fit(X_train,y_train)

sgdmodel = SGDClassifier(loss="log")
sgdmodel.fit(X_train,y_train)

# +
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import SGD, Adam
from keras.regularizers import l2

model = Sequential()

model.add(Dense(32, input_shape=(X_train.shape[1],), activation='relu')) 
model.add(Dropout(0.5))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))

model.compile(
    loss='binary_crossentropy', # loss
    optimizer="rmsprop", # learning rule
    #optimizer=Adam(), # learning rule
    metrics=['accuracy'] # show accuracy
)

print(model.summary())
# -

# Train the linear classifier on the extracted features
model.fit(x=X_train, y=y_train, batch_size=128, epochs=20, validation_split=0.2)

from sklearn.neural_network import MLPClassifier
mlp = MLPClassifier(activation="logistic")
mlp.fit(X_train,y_train)
y_pred_mlp = mlp.predict_proba(X_test)[:, 1]

from sklearn import metrics
from sklearn.metrics import roc_curve, auc

evaluation_auc(logmodel)



evaluation(logmodel.predict(X_test))


def evaluation(predictions):
    #Auswertung durch Confusion Matrix und dem Built-In Report von Sklearn
    cnf_matrix = metrics.confusion_matrix(y_test,predictions)
    print(cnf_matrix)
    print("\n")
    print(metrics.classification_report(y_test, predictions))

    class_names=[0,1] # name  of classes
    fig, ax = plt.subplots()
    tick_marks = np.arange(len(class_names))
    plt.xticks(tick_marks, class_names)
    plt.yticks(tick_marks, class_names)
    # create heatmap
    sns.heatmap(pd.DataFrame(cnf_matrix), annot=True, cmap="YlGnBu" ,fmt='g')
    ax.xaxis.set_label_position("top")
    plt.tight_layout()
    plt.title('Confusion matrix', y=1.1)
    plt.ylabel('Actual label')
    plt.xlabel('Predicted label')



def evaluation_auc(model):
    #AUC als Performanceindikator der alle möglichen Thresholds aggregiert. Je höher der AUC, also die Fläche unter der ROC-Kurve ist,
    #desto besser kann eine Klasse bestimmt werden. Jedoch ist ein AUC von 0.5 ein Indiz für eine nicht erfolgreiche Bestimmung der Klassen
    y_pred_proba = model.predict_proba(X_test)[::,1]
    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred_proba)
    auc = metrics.roc_auc_score(y_test, y_pred_proba)
    plt.plot(fpr,tpr,label="data 1, auc="+str(auc))
    plt.legend(loc=4)
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.show()
    



StarcoderdataJupyterScriptsDedupFiltered