import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import re

%matplotlib inline 

"""
Explanation: Nginx log analysis with Pandas
End of explanation
"""


pattern = re.compile(r"""(?P<remote_addr>.+?) \s-\s
            (?P<remote_user>.+?) \s
            \[(?P<time_local>.+?)\] \s 
            "(?P<method>.+?) \s (?P<uri>.+?) \s (?P<protocol>.+?)" \s
            (?P<status>.+?) \s
            (?P<body_bytes_sent>.+?) \s
            "(?P<http_referer>.+?)"? \s # Optional final " as I initially missed this when setting the format
            "(?P<http_user_agent>.*?)" \s
            (?P<request_time>.+?) \s
            (?P<upstream_response_time>.+?) \s
            (?P<pipe>.+?)""", re.VERBOSE)

test = """195.211.155.181 - - [18/Jan/2015:22:43:05 +0000] "GET /blog/email-templates-using-zend-framework/ HTTP/1.1" """ + \
        """200 7408 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Mozilla/4.0 (compatible; MSIE 6.0; """ + \
        """Windows NT 5.1; SV1) )" 0.039 0.039 ."""

m = re.match(pattern, test)
m.groupdict()

"""
Explanation: Data import - here be regex
The log format I use for nginx:
'$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'$request_time $upstream_response_time $pipe';

The following regex will match
End of explanation
"""


log_file = '/home/jonathan/dev/access.log'

with open(log_file) as f:
    log_list = []
    for i in f.readlines():
        m = re.match(pattern, i)
        log_list.append(m.groupdict())

"""
Explanation: With our working regex pattern we can parse every line in the log file and store the data in a list of dicts
End of explanation
"""


log = pd.DataFrame(log_list)

print(log.head(1))

"""
Explanation: Converting the data to a pandas DataFrame only requires passing the list to pd.DataFrame
End of explanation
"""


log.info()

"""
Explanation: Inspecting the dataframe
End of explanation
"""


log['time_local'] = pd.to_datetime(log['time_local'].apply(lambda x : x.split(' ')[0]), format='%d/%b/%Y:%H:%M:%S')

log.loc[log['upstream_response_time'] == '-', 'upstream_response_time'] = np.nan
log[['upstream_response_time', 'request_time']] = log[['upstream_response_time', 'request_time']].astype(np.float64)
log['body_bytes_sent'] = log['body_bytes_sent'].astype(np.int64)

log.info()

"""
Explanation: Currently, all columns are object type. The data will be easier to work with if the types are converted where appropriate

time_local -> datetime
upstream_response_time -> float
request_time -> float
body_bytes_sent -> integer

status is a numerical code so could be converted to an integer but as this is a categorical variable we probably don't need to convert it
End of explanation
"""


log.index = log.pop('time_local')
log.info()

"""
Explanation: The index is currently the default, an incrementing number (in this case 0 to 335084).
Setting the index to the time of the request will simplify analysis later.
End of explanation
"""


log.describe()

"""
Explanation: Basic summary information is available with the describe method
End of explanation
"""


mask = log['upstream_response_time'] > 1.0
log[mask].shape

selected_columns = ['request_time', 'status', 'upstream_response_time', 'uri']
log.loc[mask, selected_columns].sort('upstream_response_time', ascending=False)

"""
Explanation: Some of the requests take a long time. Let's take a closer look
Response time analysis
End of explanation
"""


mask = log['uri'] == '/blog/atom/'
log[mask].describe()

np.percentile(log.loc[log['uri'] == '/blog/atom/', 'upstream_response_time'], 95)

"""
Explanation: The slowest requests are on the back end but a couple of blog posts and the atom feed also appear to be slow. Are they always slow or were these requests anomalous?
End of explanation
"""


log_freq = log['body_bytes_sent'].resample('D', how='count')

log_freq.plot()

"""
Explanation: Traffic analysis
Daily requests
End of explanation
"""


top_uri = log.groupby('uri').size()
top_uri.sort()
top_uri.tail(10)

uri_mask = log['uri'] == '/blog/bcne3-search-phpbb-with-sphinx/'
selected_columns = ['remote_addr', 'method', 'http_user_agent', 'http_referer']
log.loc[mask, selected_columns].describe()

uri_mask = log['uri'] == '/blog/bcne3-search-phpbb-with-sphinx/'
method_mask = log['method'] == 'POST'
mask = uri_mask & method_mask

selected_columns = ['remote_addr', 'method', 'http_user_agent', 'http_referer']

log.loc[mask, selected_columns].describe()

"""
Explanation: Top URIs
End of explanation
"""


