# Load required packages
import numpy as np
import datetime as dt
from datetime import timedelta
import pandas as pd
from tqdm import tqdm
import os
import pkg_resources as pkg
import geopandas as gpd
from shapely.geometry import Point
from bokeh.plotting import Figure, show, output_notebook, vplot
from bokeh.charts import Bar
from bokeh.io import gridplot
from bokeh.models import HoverTool, Span, Range1d, LinearAxis
from bokeh.models.sources import ColumnDataSource
import json
import gzip
from bs4 import BeautifulSoup as bs
import requests
import stormstats
import codecs
import folium

output_notebook()

"http://en.blitzortung.org/archive_data.php?session=1&stations_users=0&selected_numbers=*&end_date=1478131200&end_time=43200&start_date=1478131200&start_time=0&rawdata_image=0&north=90&west=-180&east=180&south=-90&map=0&width_orig=640&width_result=640&agespan=60&frames=12&delay=100&last_delay=1000&show_result=1"

"""
Explanation: Lightning data analysis (from WWLN or Blitzortung)
Development notebook
This iPython notebook extracts lighning data from raw WWLN data files or Blitzortung network.
Code by: Jasa Calogovic (Faculty of Geodesy, University of Zagreb) and Benjamin Laken (UCL)
Email: jcalogovic@geof.hr
End of explanation
"""


url = "http://en.blitzortung.org/archive_data.php?session=1&stations_users=0"\
"&selected_numbers=*&end_date=1478131200&end_time=76200&start_date=1478131200&"\
"start_time=69000&rawdata_image=0&north=90&west=-180&east=180&south=-90&map=0&"\
"width_orig=640&width_result=640&agespan=60&frames=12&delay=100&last_delay=1000&show_result=1"

#csv_target ="/Tmp/archive_1_raw.txt.gz"
url

import wget

url = "http://en.blitzortung.org/Tmp/archive_2_raw.txt.gz"

fname = wget.download(url)

requests.get(url, stream=True)

#r = requests.get(url, stream=True, auth=HTTPBasicAuth('user', 'pass'))
r = requests.get(url)
soup = bs(r.text, "lxml")

r

line

for line in soup:
    print(line)

tst = r.content

bs(r.text)

stormstats.downloader.get_data(start="2015-02-01T06:30", end="2015-02-01T10:05")

stormstats.storm.gen_listfiles(data_path='./tmp_data/', ext='.gz')

import pandas as pd

import json

tmp = json.JSONDecoder('tmp_data/bz-2015-02-01-06-40.json')

json.JSONDecoder.decode('tmp_data/bz-2015-02-01-06-40.json','s')

pd.read_json('tmp_data/bz-2015-02-01-06-40.json')

"""
Explanation: Notes
It looks like you need to be an authenticated user to make a request to call back past data. BUT, after the data is called, it is added to a temporary location, at  http://en.blitzortung.org/Tmp/, which anyone can access. It seems that all files are kept there temporariliy only, and the filenames they are given (for csv type files) are archive_x_raw.txt.gz (where x is a postive integer probably corresponding to the request). So it seems that I can construct a HTTP get request via an authenticated session, and then try to download the archive file after.
I will probably have to try an algorithm to see what was the last file added to the /Tmp folder, and assume that it is mine...
End of explanation
"""


data_path='data/WWLN/Jan2016/'
#data_path = "WWLN"

# time step (in minutes) used to count number fo lightning. Max time step is 1440 (1 day)
# and it should have multiple value of 1440 minutes.
time_step=60

# if select_area=False: all lightning are counted
# if select_area=True: only lightning in selected area are counted (lat and lon limits are needed)
select_area=True
# define area to count lightning strikes (lat, lon) if select_area=True
north_lat_limit=70
south_lat_limit=30
west_lon_limit=-15
east_lon_limit=45
# exclude all lightning data with error larger than max_error 
max_error=30
# exclude all lightning strikes detected with less or equal than min_stations
min_stations=5
# note: in raw data lightning strikes detected with less than 5 stations and error larger than 30 
# are already excluded

"""
Explanation: Define parameters
End of explanation
"""


# generate list of files in data directory, if start_date and end_date are defined data only in this 
#period will be used
listfiles=gen_listfiles(data_path=data_path, ext='.loc', start_date='01-01-2016', end_date='01-02-2016')
    
# make loop for all files
i=0
for file in tqdm(listfiles):
    # read lightning data
    LN_data = read_WWLN(file=data_path+file)
    # --- make quality check and select lightning in given area
    # exclude lightning data that have larger error than max_error
    LN_data=LN_data.loc[LN_data['err']<=max_error]
    # exclude lightning data that have less than min_stations
    LN_data=LN_data.loc[LN_data['#sta']>=min_stations]
    # select only lightning strikes in given area
    if(select_area):
        LN_data=LN_data.loc[(LN_data['lat']<=north_lat_limit) & (LN_data['lat']>=south_lat_limit) & 
                            (LN_data['lon']>=west_lon_limit) & (LN_data['lon']<=east_lon_limit)]
    # --- Count lightning strikes according to time_step defined
    LN_tmp=count_lightning(LN_data, time_step)
    # add data to existing df
    if(i>=1):
        LN_count=LN_count.append(LN_tmp)
    else:
        LN_count=LN_tmp
    i=i+1

"""
Explanation: Load WWLN data and analyze it
End of explanation
"""


#LN_count
LN_count.to_csv('WWLN_data_europe.csv')

"""
Explanation: Save data
End of explanation
"""


# get data from Blitzortung server
get_data(start="2015-02-01T06:30", end="2015-02-01T10:05",
                dl_link="http://data.blitzortung.org/Data_1/Protected/Strokes/")

# 

"""
Explanation: Load data (Blitzortung)
End of explanation
"""


# define Bokeh tools
TOOLS = "pan, wheel_zoom, box_zoom, hover, reset, save"

# calculate error bars
#err_y1=LN_count['#sta_mean']-(LN_count['#sta_std']/(np.sqrt(LN_count['count'])))
#err_y2=LN_count['#sta_mean']+(LN_count['#sta_std']/(np.sqrt(LN_count['count'])))

fig_LN_count_a = Figure(plot_width=800, plot_height=400, title="Lightning count", tools=TOOLS,
                   x_axis_label="Date", y_axis_label="Nr. of lightning strikes", x_axis_type = "datetime", 
                 title_text_font_size='22pt')
fig_LN_count_a.line(LN_count['count'].index, LN_count['count'].values, color='red')

fig_LN_count_b = Figure(plot_width=800, plot_height=300, tools=TOOLS,
                 y_axis_label="Error", x_axis_type = "datetime", x_range=fig_LN_count_a.x_range)
fig_LN_count_b.line(LN_count['err_mean'].index, LN_count['err_mean'].values, color='blue')

fig_LN_count_c = Figure(plot_width=800, plot_height=300, tools=TOOLS,
                   x_axis_label="Date", y_axis_label="Mean nr. of stations", x_axis_type = "datetime",
                        x_range=fig_LN_count_a.x_range)
fig_LN_count_c.line(LN_count['#sta_mean'].index, LN_count['#sta_mean'].values, color='black')

fig_LN_count = gridplot([[fig_LN_count_a],[fig_LN_count_b],[fig_LN_count_c]])
show(fig_LN_count)

"""
Explanation: Plot lightning results
End of explanation
"""


# mapping function doesnt seem to handel the large sizes well, so I am limiting the size for now
mx = get_map(strike_data = LN_data)
mx

"""
Explanation: Map data ##
End of explanation
"""


