# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/AnandUgale/SimilarProductionDetection/blob/master/Anomaly_Detection_on_Sensor_Data.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="TMboOZrPfK9-" colab_type="code" outputId="729375a4-2103-48e1-acf0-c63b4551397b" colab={"base_uri": "https://localhost:8080/", "height": 1000}
# !pip install tensorflow==2.0

# + id="a8koFovcoFar" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="dbbea0cf-5e1d-47eb-8c56-2dc7e3e9f94a"
# install necessary packages
import tensorflow as tf
print(tf.__version__)

# + id="p14Zc30VkFre" colab_type="code" colab={}
import os
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
import joblib
import seaborn as sns
sns.set(color_codes=True)
import matplotlib.pyplot as plt
# %matplotlib inline

from numpy.random import seed
# from tensorflow import set_random_seed
# import tensorflow as tf
# tf.logging.set_verbosity(tf.logging.ERROR)

from tensorflow.keras.layers import Input, Dense, Dropout, LSTM, TimeDistributed, RepeatVector
from tensorflow.keras.models import Model
from tensorflow.keras import regularizers

# + id="KZrGz7qdkIXm" colab_type="code" colab={}
# set random seed
tf.random.set_seed(10)

# + id="yiGOYYWCkLmw" colab_type="code" outputId="5f82ef2a-21e5-472f-cc9b-2a24867db33b" colab={"base_uri": "https://localhost:8080/", "height": 276}
df = pd.read_csv('/content/data.csv')
df.info()

# + id="FupDGc1ck2xH" colab_type="code" outputId="ad31b69c-fed4-4561-dd3f-bed28af8d54a" colab={"base_uri": "https://localhost:8080/", "height": 207}
df.nunique()

# + id="UrFGyrC5k6nJ" colab_type="code" colab={}
df.drop(labels=['Timestamp', 'jobName', 'values_ConveyorBeltTimestamp', '_timestamp', '_version'], axis=1, inplace=True)

# + id="-YPEDLr5l3Do" colab_type="code" outputId="37525bca-fdcd-4a1f-bef9-362a04bcdf56" colab={"base_uri": "https://localhost:8080/", "height": 190}
df.info()

# + id="MGX1AMQUnuAr" colab_type="code" outputId="19187d5f-1fd1-4db3-98f7-0d2332f99472" colab={"base_uri": "https://localhost:8080/", "height": 198}
df.head()

# + id="YYa8NnI9n_Rk" colab_type="code" colab={}
df_new = df.groupby('Timestamp_HR', as_index=False)['values_PostStage', 'values_MidStage', 'values_PreStage'].mean()

# + id="mkSpNBdMn_lx" colab_type="code" outputId="7ca1068d-f5e1-4fe8-b15f-7e9707bd5359" colab={"base_uri": "https://localhost:8080/", "height": 173}
df_new.info()

# + id="HgBmpkV9n_pt" colab_type="code" colab={}
# set index
df_new.set_index('Timestamp_HR', inplace=True)

# + id="SmIQ9lgbl4lL" colab_type="code" outputId="9c1ca273-40f3-41f8-e176-2ae678d9bed5" colab={"base_uri": "https://localhost:8080/", "height": 155}
df_new.info()

# + id="soWNYsj9oRh7" colab_type="code" outputId="d7a5f427-f9b8-440e-8be5-2cfafb72a827" colab={"base_uri": "https://localhost:8080/", "height": 228}
df_new.head()

# + id="M4zbKP7ppfx7" colab_type="code" colab={}
df_new.to_csv('sensor.csv')

# + id="0AQjkdcOpo56" colab_type="code" colab={}
# Define train and test dataset
df_new.drop(labels=['values_MidStage', 'values_PreStage'], axis=1, inplace=True)

# + id="LMBn8XKaDT1J" colab_type="code" outputId="f67a5970-aab6-49b0-e45b-cada273f7182" colab={"base_uri": "https://localhost:8080/", "height": 228}
df_new.head()

# + id="aSoWAm2Tt1e4" colab_type="code" colab={}
# split into train and test split
train = df_new[:2162]
val = df_new[2162:]

# + id="9_wCrlcIuWyn" colab_type="code" outputId="a2c97ad3-2189-47c8-dcc6-5c5bc206b78d" colab={"base_uri": "https://localhost:8080/", "height": 34}
train.shape, val.shape

# + id="F9SbGERGuh9a" colab_type="code" outputId="f2a99e68-fb50-4d8d-8c4c-0be0badb362c" colab={"base_uri": "https://localhost:8080/", "height": 439}
# Only fot values_PostStage
fig, ax = plt.subplots(figsize=(14, 6), dpi=80)
ax.plot(train['values_PostStage'], label='values_PostStage', color='blue', animated = True, linewidth=1)
plt.legend(loc='lower left')
ax.set_title('Sensor Training Data', fontsize=16)
plt.show()

# + id="VXRorA1P9DXF" colab_type="code" outputId="a9de549d-7c9b-45b2-a409-15803978c437" colab={"base_uri": "https://localhost:8080/", "height": 318}
# Only fot values_PostStage
fig, ax = plt.subplots(figsize=(10, 4), dpi=80)
ax.plot(val['values_PostStage'], label='values_PostStage', color='blue', animated = True, linewidth=1)
plt.legend(loc='lower left')
ax.set_title('Sensor Training Data', fontsize=16)
plt.show()

# + id="Zo8-hRaH9lMd" colab_type="code" colab={}
# transforming data from the time domain to the frequency domain using fast Fourier transform
train_fft = np.fft.fft(train)
val_fft = np.fft.fft(val)

# + id="ehx4sMyq9wW5" colab_type="code" outputId="68f8b5d0-a14c-4aef-85e3-6810d81ad46a" colab={"base_uri": "https://localhost:8080/", "height": 977}
# frequencies of the healthy sensor signal
fig, ax = plt.subplots(figsize=(14, 10), dpi=80)
ax.plot(train_fft[:,0], label='values_PostStage', color='blue', animated = True, linewidth=1)
plt.legend(loc='top left')
ax.set_title('Sensor Training Data', fontsize=16)
plt.show()

# + id="GSabHhF--Klq" colab_type="code" outputId="518d56d8-1a42-47d3-dc90-41a0df512e18" colab={"base_uri": "https://localhost:8080/", "height": 715}
# frequencies of the healthy sensor signal
fig, ax = plt.subplots(figsize=(14, 10), dpi=80)
ax.plot(val_fft[:,0], label='values_PostStage', color='blue', animated = True, linewidth=1)
plt.legend(loc='lower left')
ax.set_title('Sensor Training Data', fontsize=16)
plt.show()

# + id="NStXQexz-gGk" colab_type="code" outputId="2b43341e-cda8-4067-e109-a4b2369e7cd7" colab={"base_uri": "https://localhost:8080/", "height": 34}
# first normalize it to a range between 0 and 1.
# normalize the data
scaler = MinMaxScaler()
X_train = scaler.fit_transform(train)
X_val = scaler.transform(val)
scaler_filename = "scaler_data"
joblib.dump(scaler, scaler_filename)

# + id="-4L158SGCzQb" colab_type="code" outputId="ce6f362d-c1b2-4c10-f00e-adbb225999c7" colab={"base_uri": "https://localhost:8080/", "height": 34}
X_train.shape

# + id="bGRkwG1qDiTk" colab_type="code" outputId="8fb3767d-2242-4dca-a8b1-d9839e35747b" colab={"base_uri": "https://localhost:8080/", "height": 34}
X_val.shape

# + id="qYNu24gNCquB" colab_type="code" outputId="76ad173d-7602-43a1-bd3a-d16ce06e49df" colab={"base_uri": "https://localhost:8080/", "height": 52}
# reshape inputs for LSTM [samples, timesteps, features]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])
print("Training data shape:", X_train.shape)
X_val = X_val.reshape(X_val.shape[0], 1, X_val.shape[1])
print("Test data shape:", X_val.shape)


# + id="wXL72BLxDqJJ" colab_type="code" colab={}
# define the autoencoder network model
def autoencoder_model(X):
    inputs = Input(shape=(X.shape[1], X.shape[2]))
    L1 = LSTM(16, activation='relu', return_sequences=True, 
              kernel_regularizer=regularizers.l2(0.00))(inputs)
    L2 = LSTM(4, activation='relu', return_sequences=False)(L1)
    L3 = RepeatVector(X.shape[1])(L2)
    L4 = LSTM(4, activation='relu', return_sequences=True)(L3)
    L5 = LSTM(16, activation='relu', return_sequences=True)(L4)
    output = TimeDistributed(Dense(X.shape[2]))(L5)    
    model = Model(inputs=inputs, outputs=output)
    return model


# + id="82bpQ1asDu1L" colab_type="code" outputId="203447a8-a773-4f5c-dc97-6c50851c1eb8" colab={"base_uri": "https://localhost:8080/", "height": 397}
# create the autoencoder model
model = autoencoder_model(X_train)
model.compile(optimizer='adam', loss='mae')
model.summary()

# + id="AeM6e8XDDyGd" colab_type="code" outputId="02c769aa-3c87-4cb7-db9d-a593081f9d51" colab={"base_uri": "https://localhost:8080/", "height": 1000}
# fit the model to the data
nb_epochs = 30
batch_size = 100
history = model.fit(X_train, X_train, epochs=nb_epochs, batch_size=batch_size,
                    validation_split=0.05).history

# + id="Emf4BOUNEJyu" colab_type="code" outputId="c8ecf847-8483-457f-b9cc-7a31ef0f611e" colab={"base_uri": "https://localhost:8080/", "height": 268}
plt.plot(history['loss'], label='train')
plt.plot(history['val_loss'], label='val')
plt.legend();

# + id="a6tIpM0dEsJX" colab_type="code" colab={}
# Finding Anomalies
X_train_pred = model.predict(X_train)

train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)

# + id="rWsoIKZyFOJq" colab_type="code" outputId="834a8075-7ee3-455a-f4f1-04f5b1965c09" colab={"base_uri": "https://localhost:8080/", "height": 268}
# look at the error:
sns.distplot(train_mae_loss, bins=50, kde=True);

# + id="Xhhyj-B9FPtE" colab_type="code" colab={}
THRESHOLD = 0.03

# + id="YUjWnQr1FZMy" colab_type="code" colab={}
# Let’s calculate the MAE on the test data:
X_val_pred = model.predict(X_val)

val_mae_loss = np.mean(np.abs(X_val_pred - X_val), axis=1)

# + id="yeZ9rvWwFhIu" colab_type="code" colab={}
# We’ll build a DataFrame containing the loss and the anomalies (values above the threshold):
val_score_df = pd.DataFrame(index=val.index)
val_score_df['loss'] = val_mae_loss
val_score_df['threshold'] = THRESHOLD
val_score_df['anomaly'] = val_score_df.loss > val_score_df.threshold
val_score_df['values_PostStage'] = val.values_PostStage

# + id="43zpWR_jF_hl" colab_type="code" outputId="d56fc438-2f77-437f-9149-b1875d4f3ae4" colab={"base_uri": "https://localhost:8080/", "height": 307}
plt.plot(val_score_df.index, val_score_df.loss, label='loss')
plt.plot(val_score_df.index, val_score_df.threshold, label='threshold')
plt.xticks(rotation=25)
plt.legend();

# + id="0G2Uyq-oFwna" colab_type="code" outputId="1078c5d3-0713-45ed-da9c-c07ba55bf59d" colab={"base_uri": "https://localhost:8080/", "height": 228}
# Looks like we’re thresholding extreme values quite well. Let’s create a DataFrame using only those:
anomalies = val_score_df[val_score_df.anomaly == True]
anomalies.head()

# + id="87EhLWFtHBVO" colab_type="code" outputId="e746e8b4-7d31-4b23-9036-9945004d2343" colab={"base_uri": "https://localhost:8080/", "height": 324}
plt.plot(
  val.index, 
  val.values_PostStage, 
  label='values_PostStage'
);

sns.scatterplot(
  anomalies.index,
  anomalies.values_PostStage,
  color=sns.color_palette()[3],
  s=52,
  label='anomaly'
)
plt.xticks(rotation=25)
plt.legend();

# + id="S2L_r7y_I26A" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered