# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import sys
import os
import json
import pandas
import numpy
import optparse
from keras.callbacks import TensorBoard
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.preprocessing.text import Tokenizer
from collections import OrderedDict
import json

csv_file = 'data/dev-access.csv'

# +
dataframe = pandas.read_csv(csv_file, engine='python', quotechar='|', header=None)
count_frame = dataframe.groupby([1]).count()
print(count_frame)
total_req = count_frame[0][0] + count_frame[0][1]
num_malicious = count_frame[0][1]

print("Malicious request logs in dataset: {:0.2f}%".format(float(num_malicious) / total_req * 100))
# -

dataset = dataframe.sample(frac=1).values

dataframe.head()

for i in range(len(dataframe[0])):
    dataframe[0][i] = dict(json.loads(dataframe[0][i]))
    if i%1000 == 0:
        print(i, end=", ")

# +
# dataframe.apply(lambda x: json.loads(str(x)), axis=1)
# -

dataframe.head()

dataframe.loc[344][0]

datarows = []

for i in range(len(dataframe[0])):
    frame = dataframe[0][i]
    res = dataframe[1][i]
    
    method = frame['method']
    path = frame['path']
    query = frame['query']
    if len(query) > 0:
        query = query['query']
    statusCode = frame['statusCode']
    source = frame['source']['remoteAddress']
    route = frame['route']
    referer = ""
    if "referer" in frame['source'].keys():
        referer = frame['source']['referer']
    response = frame['responsePayload']
    error = ""
    msg = ""
    if(type(response) == type("hello")):
        msg = response
    else:
        if "error" in response.keys():
            error = response['error']
        msg = response['message']
    
    datarows.append([method, path, query, statusCode, source, referer, route, error, msg, res])
    

with open("data/data.csv", "w", newline="") as f:
    for i in range(len(datarows)):
        f.write(",".join(map(str, datarows[i])))
        f.write("\n")



StarcoderdataJupyterScriptsDedupFiltered