# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
##################################################################
#   Author: <NAME> 
#
##################################################################
#   Network Traffic Anomaly Intrusion Detection System (IDS) 
#   using Machine Learning
##################################################################


#This commented out code was used to name all the data features and process the
# data file into CSV
'''
df = pd.read_csv("android_traffic.csv", sep=";", header=0)
df
df.to_csv("android_traffic.csv")
print(len(df.index))
'''

#test dataset
''''
test_dataset = pd.read_csv("Test.txt",sep=",",header=None)
test_dataset.columns = ["duration","protocol_type","service","flag","src_bytes","dst_bytes","land",
"wrong_fragment","urgent","hot","num_failed_logins","logged_in",
"num_compromised","root_shell","su_attempted","num_root","num_file_creations",
"num_shells","num_access_files","num_outbound_cmds","is_host_login",
"is_guest_login","count","srv_count","serror_rate", "srv_serror_rate",
"rerror_rate","srv_rerror_rate","same_srv_rate", "diff_srv_rate", "srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate",
"dst_host_diff_srv_rate","dst_host_same_src_port_rate",
"dst_host_srv_diff_host_rate","dst_host_serror_rate","dst_host_srv_serror_rate",
"dst_host_rerror_rate","dst_host_srv_rerror_rate","attack", "last_flag"]
test_dataset.to_csv("testTraffic.csv", index=False)
print(len(test_dataset.index))

#train dataset

train_dataset = pd.read_csv("Train.txt", sep=",", header=None)
train_dataset.columns=["duration","protocol_type","service","flag","src_bytes","dst_bytes","land",
"wrong_fragment","urgent","hot","num_failed_logins","logged_in",
"num_compromised","root_shell","su_attempted","num_root","num_file_creations",
"num_shells","num_access_files","num_outbound_cmds","is_host_login",
"is_guest_login","count","srv_count","serror_rate", "srv_serror_rate",
"rerror_rate","srv_rerror_rate","same_srv_rate", "diff_srv_rate", "srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate",
"dst_host_diff_srv_rate","dst_host_same_src_port_rate",
"dst_host_srv_diff_host_rate","dst_host_serror_rate","dst_host_srv_serror_rate",
"dst_host_rerror_rate","dst_host_srv_rerror_rate","attack", "last_flag"]

train_dataset.to_csv("trainTraffic.csv", index=False)
'''



# +
train_df = pd.read_csv("trainTraffic.csv")

#Training data description (count, mean,std,min, 25-75 percentile)
print("Training Dataset Description (count, mean, std, min, etc)")
train_df.describe()
 
# -

print("Training Dataset")
train_df

#Rename the column "attack" to "label", this is the target column/ground truth
train_df=train_df.rename(columns={"attack":"label"})


#View value counts of target label - get an idea of data distribution
print("Value counts of each label")
train_df['label'].value_counts()


print("Number of labels for target : %s" %len(train_df['label'].value_counts()))
print("Number of features for dataset : %s" %len(train_df.columns))

# +
#Place all numeric labels to a list
nfeatures = train_df.columns.tolist()

#Remove categorical data features, and target column 
nfeatures.remove('label')
nfeatures.remove('protocol_type')
nfeatures.remove('service')
nfeatures.remove('flag')

#Create dataframe with only numerical features as float type
data_features = train_df[nfeatures].astype(float)

# -

#EDA of numerica features
data_features.describe()



'''
#Change target labels to binary labels - normal or malicious
binary_labels = train_df['label'].copy()
binary_labels[binary_labels != 'normal'] = 'malicious'
binary_labels.value_counts()
'''

train_df.columns

'''
#This is for binary classification
len(train_df['label'].unique())
#Convert all non-normal labels to malicious - turn into binary classification
train_df.loc[:]['label'][train_df['label'] != 'normal'] = 'malicious'
print("Train dataframe with binary class labels")
train_df
'''

# +
import sklearn.model_selection as sk

X_train, X_test, y_train, y_test = sk.train_test_split(data_features, train_df['label'], test_size=0.2,random_state=42)

from sklearn import preprocessing
import numpy as np

#Scale the data
minmax_scale = preprocessing.MinMaxScaler().fit(X_train)
X_train_scaled = minmax_scale.transform(X_train)
X_test_scaled = minmax_scale.transform(X_test)
print(np.shape(X_train_scaled))

# +
import sklearn.naive_bayes 
from time import time
from sklearn.neighbors import KNeighborsClassifier

#k-NN with 5 nearest neighbors using "ball_tree" 500 leaf
classifier = KNeighborsClassifier(n_neighbors = 5, algorithm = 'ball_tree', leaf_size = 500)
start = time()
classifier.fit(X_train_scaled, y_train)
end = time() - start
print ("Classifier trained in {} seconds.".format(round(end, 3)))

#Predictions on the X_test
start = time()
pred = classifier.predict(X_test_scaled)
end = time() - start 
print ("Predicted {} samples in {} seconds".format(len(pred),round(end,3)))



# +
#Calculating out the accuracy
from sklearn.metrics import accuracy_score


acc = accuracy_score(pred, y_test)
print(acc)
print(pred)
print(y_test)
print ("Accuracy on train dataset is {}.".format(round(acc,4)))
# -





# +
# Now scale, train, predict on "testTraffic.csv"
test_df = pd.read_csv("testTraffic.csv")
test_df=test_df.rename(columns={"attack":"label"})

print("Number of columns %d" % len(test_df.columns))
test_df

# +
#Same processing of dataframe before - remove categorical features
#Place all numeric labels to a list
nfeatures = test_df.columns.tolist()

#Remove categorical data features, and target column 
nfeatures.remove('label')
nfeatures.remove('protocol_type')
nfeatures.remove('service')
nfeatures.remove('flag')

#Create dataframe with only numerical features as float type
test_data_features = test_df[nfeatures].astype(float)
# -

#EDA of numerica features
test_data_features.describe()

'''
#This is for binary classification
len(test_df['label'].unique())
#Convert all non-normal labels to malicious - turn into binary classification
test_df.loc[:]['label'][test_df['label'] != 'normal'] = 'malicious'
test_df
'''

test_data_features

# +
#Run same split, scaling, classifier, prediction on "testTraffic.csv"

#99% of samples are for test 
X_train, X_test, y_train, y_test = sk.train_test_split(test_data_features, test_df['label'], test_size=22543 ,random_state=42)


#Scale the data
minmax_scale = preprocessing.MinMaxScaler().fit(X_test)
X_train_scaled = minmax_scale.transform(X_train)
X_test_scaled = minmax_scale.transform(X_test)
print("Number of samples removed from dataset {}".format(np.shape(X_train_scaled)[0]))


start = time()
#We dont need to train the classifier again as its been trained w/ the train dataset already
pred = classifier.predict(X_test_scaled)
acc = accuracy_score(pred, y_test)
end = time() - start
print ("Accuracy on test dataset is {}.".format(round(acc,4)))
print ("Predicted (already trained classifier) {} samples in {} seconds".format(len(pred),round(end,3)))
# -





StarcoderdataJupyterScriptsDedupFiltered