# -*- coding: utf-8 -*-


# Create your views here.
from incidents.models import IncidentCategory, Incident, Comments, BusinessLine, Label, Artifact, File, Log, BaleCategory, Attribute, ValidAttribute, IncidentTemplate, Profile
from incidents.models import IncidentForm, CommentForm
from fir_artifacts.hash import Hash

from django.conf import settings
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required, user_passes_test
from django.core import serializers
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.servers.basehttp import FileWrapper
from django.core.urlresolvers import reverse
from django.db.models import Q, Max
from django.http import HttpResponse, HttpResponseServerError, Http404
from django.shortcuts import get_object_or_404, render, redirect
from django.template import RequestContext
from json import dumps
from django.template import Template
from django.core.exceptions import ObjectDoesNotExist
from django.forms.models import model_to_dict

import os, tempfile, zipfile, re, mimetypes, datetime
from dateutil.relativedelta import *
from dateutil.parser import parse
from bson import json_util

import copy
import math

from fir_artifacts import artifacts as libartifacts

cal = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']

# helper =========================================================


def is_incident_handler(user):
	return user.has_perm('incidents.handle_incidents')


def can_view_statistics(user):
	return user.has_perm('incidents.view_statistics') or user.has_perm('incidents.handle_incidents')

# login / logout =================================================


def user_login(request):
	if request.method == "POST":
		username = request.POST['username']
		password = request.POST['password']
		user = authenticate(username=username, password=password)
		if user is not None:
			if not request.POST.get('remember', None):
				request.session.set_expiry(0)

			try:
				Profile.objects.get(user=user)
			except ObjectDoesNotExist, e:
				profile = Profile()
				profile.user = user
				profile.hide_closed = False
				profile.incident_number = 50
				profile.save()

			if user.is_active:
				login(request, user)
				log("Login", user)
				init_session(request)
				return redirect('dashboard:main')
			else:
				return HttpResponse('Account disabled')
		else:
			return render(request, 'incidents/login.html', {'error': 'error'})
	else:
		return render(request, 'incidents/login.html')


def user_logout(request):
	logout(request)
	request.session.flush()
	return redirect('login')


def init_session(request):
	pass
	# Put all the incident templates in the session
	request.session['incident_templates'] = list(IncidentTemplate.objects.exclude(name='default').values('name'))
	request.session['has_incident_templates'] = len(request.session['incident_templates']) > 0

# audit trail =====================================================


def log(what, user, incident=None, comment=None):
	# dirty hack to not log when in debug mode
	import sys
	if len(sys.argv) >= 2 and sys.argv[1] == 'runserver':
		print "DEBUG: Not logging"
		return

	log = Log()
	log.what = what
	log.who = user
	log.incident = incident
	log.comment = comment

	log.save()

# incidents =======================================================


@login_required
@user_passes_test(is_incident_handler)
def followup(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)
	comments = i.comments_set.all().order_by('date')

	return render(request, 'incidents/followup.html', {'incident': i, 'comments': comments})


@login_required
@user_passes_test(is_incident_handler)
def index(request, is_incident=False):
	return render(request, 'events/index-all.html', {'incident_view': is_incident})


@login_required
@user_passes_test(is_incident_handler)
def incidents_all(request):
	return incident_display(request, Q(is_incident=True))


@login_required
@user_passes_test(is_incident_handler)
def events_all(request):
	return incident_display(request, Q(is_incident=False), False)


@login_required
@user_passes_test(is_incident_handler)
def details(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)

	form = CommentForm()

	(artifacts, artifacts_count, correlated_count) = libartifacts.all_for_event(i)

	valid_attributes = i.category.validattribute_set.all()
	attributes = i.attribute_set.all()

	comments = i.comments_set.all().order_by('date')

	return render(request, "events/detail-all.html", {	"event": i,
														"comment_form": form,
														'correlated_count': correlated_count,
														'artifacts_count': artifacts_count,
														'artifacts': artifacts,
														'attributes': attributes,
														'valid_attributes': valid_attributes,
														'comments' : comments
													})


@login_required
@user_passes_test(is_incident_handler)
def new_event(request):
	if request.method == 'POST':
		form = IncidentForm(request.POST)
		form.status = 'Open'

		if form.is_valid():
			i = form.save(commit=False)

			if form.cleaned_data['is_major'] == False:
				i.is_major = form.cleaned_data['category'].is_major

			if i.is_major:
				i.is_incident = True

			comment = Comments()
			comment.comment = "Incident opened"
			comment.action = Label.objects.get(name='Opened')
			i.opened_by = request.user
			i.save()
			form.save_m2m()
			i.refresh_main_business_lines()
			comment.incident = i
			comment.opened_by = request.user
			comment.date = i.date
			comment.save()

			# logging
			log("Created incident", request.user, incident=i)

			i.refresh_artifacts(i.description)

			if i.is_incident:
				return redirect("incidents:details", incident_id=i.id)
			else:
				return redirect("events:details", incident_id=i.id)

	else:
		form = IncidentForm()

		template = request.GET.get('template', 'default')
		try:
			template = IncidentTemplate.objects.get(name=template)
			data = model_to_dict(template)
			data['description'] = Template(data['description']).render(RequestContext(request))
			form = IncidentForm(initial=data)
		except ObjectDoesNotExist:
			pass

	bls = BusinessLine.objects.all()

	return render(request, 'events/new.html', {'form': form, 'mode': 'new', 'bls': bls})


def diff(incident, form):
	comments = []
	for i in form:
		# skip the following fields from diff
		if i in ['description', 'concerned_business_lines', 'main_business_lines']:
			continue

		new = form[i]
		old = getattr(incident, i)

		if new != old:

			label = i

			if i == 'is_major':
				label = 'major'
			if i == 'concerned_business_lines':
				label = "business lines"
			if i == 'main_business_line':
				label = "main business line"
			if i == 'is_incident':
				label = 'incident'

			if old == "O": old = 'Open'
			if old == "C": old = 'Closed'
			if old == "B": old = 'Blocked'
			if new == "O": new = 'Open'
			if new == "C": new = 'Closed'
			if new == "B": new = 'Blocked'

			comments.append(u'Changed "%s" from "%s" to "%s";' % (label, old, new))

	return "\n".join(comments)


@login_required
@user_passes_test(is_incident_handler)
def edit_incident(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)
	starred = i.is_starred
	bls = BusinessLine.objects.all()

	if request.method == "POST":
		form = IncidentForm(request.POST, instance=i)

		if form.is_valid():
			extra_comments = diff(Incident.objects.get(pk=incident_id), form.cleaned_data)
			if extra_comments != "":
				c = Comments()
				c.comment = extra_comments
				c.action = Label.objects.get(name='Info')
				c.incident = i
				c.opened_by = request.user
				c.save()

				log("Edit incident", request.user, incident=i)

			# update main BL
			form.save()
			i.refresh_main_business_lines()
			i.is_starred = starred
			i.refresh_artifacts(i.description)
			i.save()

			if i.is_incident:
				return redirect("incidents:details", incident_id=i.id)
				return redirect("incidents:index")
			else:
				return redirect("events:details", incident_id=i.id)
				return redirect("events:index")
	else:
		form = IncidentForm(instance=i)

	return render(request, 'events/new.html', {'i': i, 'form': form, 'mode': 'edit', 'bls': bls})


@login_required
@user_passes_test(is_incident_handler)
def delete_incident(request, incident_id):
	if request.method == "POST":
		i = get_object_or_404(Incident, pk=incident_id)
		msg = "Incident '%s' deleted." % i.subject
		i.delete()
		return HttpResponse(msg)
	else:
		return redirect("incidents:index")


@login_required
@user_passes_test(is_incident_handler)
def change_status(request, incident_id, status):
	i = get_object_or_404(Incident, pk=incident_id)
	i.status = status
	i.save()

	status_name = 'Closed'
	if status == 'O':
		status_name = 'Opened'
	elif status == 'B':
		status_name = 'Blocked'

	c = Comments()
	c.comment = "Status changed to '%s'" % status_name
	c.date = datetime.datetime.now()
	c.action = get_object_or_404(Label, name='Closed', group__name='action')
	c.incident = i
	c.opened_by = request.user
	c.save()

	return redirect('dashboard:main')


# file management ===============================================================


@login_required
@user_passes_test(is_incident_handler)
def upload_file(request, incident_id):

	if request.method == 'POST':
		incident = get_object_or_404(Incident, pk=incident_id)
		descriptions = request.POST.getlist('description')
		files = request.FILES.getlist('file')

		if len(descriptions) == len(files):  # consider this as a valid upload form?
			for i, file in enumerate(files):
				handle_uploaded_file(file, descriptions[i], incident)
				log("Uploaded file %s" % file.name[:80], request.user, incident=incident)

	return redirect("incidents:details", incident_id=incident_id)


def handle_uploaded_file(file, description, incident):

	f = File()
	f.description = description
	f.file = file
	f.incident = incident
	f.file.name = re.sub('[^\w\.\-]', '_', f.file.name)
	f.save()

	hashes = f.get_hashes()
	for h in hashes:
		print "Got %s hash: %s " % (h, hashes[h])
		try:
			a = Artifact.objects.get(value=hashes[h])
			a.save()
		except Exception:
			a = Artifact()
			a.type = Hash.key
			a.value = hashes[h]
			a.save()

		a.incidents.add(incident)
		f.hashes.add(a)
	f.save()


@login_required
@user_passes_test(is_incident_handler)
def download(request, incident_id, filename):
	f = get_object_or_404(File, incident_id=incident_id, file="%s/%s" % (incident_id, filename))

	wrapper = FileWrapper(f.file)
	content_type = mimetypes.guess_type(f.file.name)
	response = HttpResponse(wrapper, content_type=content_type)
	response['Content-Disposition'] = 'attachment; filename=%s' % (f.getfilename())
	response['Content-Length'] = os.path.getsize(str(f.file.file))

	return response


@login_required
@user_passes_test(is_incident_handler)
def download_archive(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)
	if i.file_set.count() == 0:
		raise Http404
	temp = tempfile.TemporaryFile()
	archive = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)
	path = settings.MEDIA_ROOT + "/%s/" % incident_id

	for filename in os.listdir(path):
		archive.write(path+filename, os.path.basename(filename))
	archive.close()
	wrapper = FileWrapper(temp)
	response = HttpResponse(wrapper, content_type='application/zip')
	response['Content-Disposition'] = 'attachment; filename=%s-archive.zip' % (i.subject)
	response['Content-Length'] = temp.tell()
	temp.seek(0)
	return response

	pass


@login_required
@user_passes_test(is_incident_handler)
def remove_file(request, incident_id, file_id):
	if request.method == "POST":
		f = get_object_or_404(File, pk=file_id, incident_id=incident_id)
		filename = f.file.name
		f.file.delete()
		f.delete()
		log("Deleted file %s" % filename[:70], request.user, incident=Incident.objects.get(id=incident_id))
		return redirect('incidents:details', incident_id=f.incident_id)
	else:
		return redirect('incidents:details', incident_id=f.incident_id)


# attributes ================================================================


@login_required
@user_passes_test(is_incident_handler)
def add_attribute(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)
	if request.method == "POST":
		# First, check if it is a valid attribute
		valid_attribute = get_object_or_404(ValidAttribute, name=request.POST['name'])

		# Create a new attribute
		a = Attribute(name=valid_attribute.name, value=request.POST['value'])
		# Except if valid attribute has an unit and this particular attribute already exists
		# In this case, a single attribute should be keeped, with an updated value
		if valid_attribute.unit is not None and valid_attribute.unit != "":
			try:
				a = i.attribute_set.get(name=valid_attribute.name)
				a.value = str(int(a.value) + int(request.POST['value']))
			except:
				pass

		a.incident = i
		a.save()

		if request.is_ajax():
			return render(request, 'events/_attributes.html', {'attributes': i.attribute_set.all(), 'event': i})

	return redirect('incidents:details', incident_id=incident_id)


@login_required
@user_passes_test(is_incident_handler)
def delete_attribute(request, incident_id, attribute_id):
	a = get_object_or_404(Attribute, pk=attribute_id)
	if request.method == "POST":
		a.delete()
	return redirect('incidents:details', incident_id=incident_id)


# comments ==================================================================

@login_required
@user_passes_test(is_incident_handler)
def edit_comment(request, incident_id, comment_id):
	c = get_object_or_404(Comments, pk=comment_id, incident_id=incident_id)
	if request.method == "POST":
		form = CommentForm(request.POST, instance=c)
		if form.is_valid():
			form.save()
			log("Edited comment %s" % (form.cleaned_data['comment'][:10]+"..."), request.user, incident=Incident.objects.get(id=incident_id))
			return redirect("incidents:details", incident_id=c.incident_id)
	else:
		form = CommentForm(instance=c)

	return render(request, 'events/edit_comment.html', {'c': c, 'form': form})


@login_required
@user_passes_test(is_incident_handler)
def delete_comment(request, incident_id, comment_id):

	if request.method == "POST":
		c = get_object_or_404(Comments, pk=comment_id, incident_id=incident_id)
		msg = "Comment '%s' deleted." % (c.comment[:20]+"...")
		c.delete()
		log(msg, request.user, incident=Incident.objects.get(id=incident_id))
		return redirect('incidents:details', incident_id=c.incident_id)
	else:
		return redirect('incidents:details', incident_id=c.incident_id)


@login_required
@user_passes_test(is_incident_handler)
def update_comment(request, comment_id):

	if request.method == 'GET':
		c = [get_object_or_404(Comments, pk=comment_id)]
		serialized = serializers.serialize('json', c)
		return HttpResponse(dumps(serialized), content_type="application/json")
	else:
		c = get_object_or_404(Comments, pk=comment_id)
		comment_form = CommentForm(request.POST, instance=c)

		if comment_form.is_valid():

			c = comment_form.save()

			log("Comment edited: %s" % (comment_form.cleaned_data['comment'][:20]+"..."), request.user, incident=c.incident)

			if c.action.name in ['Closed', 'Opened', 'Blocked']:
				c.incident.status = c.action.name[0]
				c.incident.save()

			i = get_object_or_404(Incident, pk=c.incident.id)
			i.refresh_artifacts(c.comment)

			return render(request, 'events/_comment.html', {'comment': c, 'event': i})
		else:
			ret = {'status': 'error', 'errors': comment_form.errors}
			return HttpResponseServerError(dumps(ret), content_type="application/json")


# events ====================================================================

@login_required
@user_passes_test(is_incident_handler)
def event_index(request):
	return index(request, False)


def normalize_query(query_string,
					findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
					normspace=re.compile(r'\s{2,}').sub):
	''' Splits the query string in invidual keywords, getting rid of unecessary spaces
		and grouping quoted words together.
		Example:

		>>> normalize_query('  some random  words "with   quotes  " and   spaces')
		['some', 'random', 'words', 'with quotes', 'and', 'spaces']

	'''
	return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)]


def get_query(query_string, search_fields):
	''' Returns a query, that is a combination of Q objects. That combination
		aims to search keywords within a model by testing the given search fields.

	'''
	query = None  # Query to search for every search term
	terms = normalize_query(query_string)
	for term in terms:
		or_query = None  # Query to search for a given term in each field
		for field_name in search_fields:
			q = Q(**{"%s__icontains" % field_name: term})

			if or_query is None:
				or_query = q
			else:
				or_query = or_query | q
		if query is None:
			query = or_query
		else:
			query = query & or_query
	return query


@login_required
@user_passes_test(is_incident_handler)
def search(request):
	query_string = ''
	if ('q' in request.GET) and request.GET['q'].strip():
		query_string = request.GET['q']

		if request.is_ajax():
			asc = request.GET.get('asc', 'false')
			q = Q()

			plan = re.search("plan:(\S+)", query_string)
			if plan:
				q = q & Q(plan__name=plan.group(1))
				query_string = query_string.replace("plan:"+plan.group(1), '')

			try:
				bl = re.search("bl:(\S+)", query_string).group(1)
				bls = BusinessLine.objects.filter(name__icontains=bl)
				if bls:
					q = q & (Q(concerned_business_lines__in=bls) | Q(main_business_lines__in=bls))
					query_string = query_string.replace("bl:"+bl, '')
			except Exception:
				pass

			category = re.search("category:(\S+)", query_string)
			if category:
				q = q & Q(category__name__icontains=category.group(1))
				query_string = query_string.replace('category:'+category.group(1), '')

			status = re.search("status:(\S+)", query_string)
			if status:
				q = q & Q(status=status.group(1)[0])
				query_string = query_string.replace('status:'+status.group(1), '')

			if query_string.count('starred') > 0:
				q = q & Q(is_starred=True)
				query_string = query_string.replace('starred', '')

			severity = re.search("severity(?P<eval>[:<>])(?P<value>\d)", query_string)
			if severity:
				if severity.group('eval') == ':':
					q = q & Q(severity=severity.group("value"))
				elif severity.group('eval') == ">":
					q = q & Q(severity__gt=severity.group("value"))
				elif severity.group('eval') == "<":
					q = q & Q(severity__lt=severity.group("value"))
				query_string = query_string.replace('severity'+severity.group('eval')+severity.group("value"), '')

			pattern = re.compile('\s+')

			query_string = query_string.strip()

			other = pattern.split(query_string)
			if query_string != ['']:
				q_other = Q()
				for i in other:
					q_other |= (Q(subject__icontains=i) | Q(description__icontains=i) | Q(comments__comment__icontains=i))

			q = (q & q_other)

			# TODO a function that takes in incidents and returns them ordered

			order_param = request.GET.get('order_by', 'date')

			order_by = order_param

			if order_by not in ['date', 'subject', 'category', 'bl', 'severity', 'status', 'opened_by', 'detection', 'actor', 'confidentiality']:
				order_by = 'date'

			if order_by == "category": order_by = 'category__name'
			if order_by == 'detection': order_by = 'detection__name'
			if order_by == 'actor': order_by = 'actor__name'

			if asc == 'false':
				order_by = "-"+order_by

			if order_param == 'last_action':
				if asc:
					found_entries = Incident.objects.filter(q).annotate(Max('comments__date')).order_by('comments__date__max')
				else:
					found_entries = Incident.objects.filter(q).annotate(Max('comments__date')).order_by('-comments__date__max')

			else:
				found_entries = Incident.objects.filter(q).all().order_by(order_by)

			# distinct
			found_entries = found_entries.distinct()

			# get hide_closed option from user profile
			if request.user.profile.hide_closed:
				found_entries = found_entries.filter(~Q(status='C'))

			# get number of pages from user profile
			page = request.GET.get('page')
			incident_number = request.user.profile.incident_number

			p = Paginator(found_entries, incident_number)

			try:
				found_entries = p.page(page)
			except PageNotAnInteger:
				found_entries = p.page(1)
			except EmptyPage:
				found_entries = p.page(1)

			return render(request, 'events/table.html', {'incident_list': found_entries, 'order_param': order_param, 'asc': asc})
		else:
			return render(request, 'events/search.html', {'query_string': query_string})
	else:
		return redirect('incidents:index')

	if request.is_ajax():
		query_string = ''
		found_entries = None
		if ('q' in request.GET) and request.GET['q'].strip():
			query_string = request.GET['q']
		else:
			return redirect('incidents:index')
	else:
		return render(request, 'events/search.html', {'query_string': q})


# ajax ======================================================================

@login_required
@user_passes_test(is_incident_handler)
def toggle_star(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)

	i.is_starred = not i.is_starred

	i.save()

	return HttpResponse(dumps({'starred': i.is_starred}), content_type='application/json')


@login_required
@user_passes_test(is_incident_handler)
def comment(request, incident_id):
	i = get_object_or_404(Incident, pk=incident_id)

	if request.method == "POST":
		comment_form = CommentForm(request.POST)
		if comment_form.is_valid():
			com = comment_form.save(commit=False)
			com.incident = i
			com.opened_by = request.user
			com.save()
			log("Comment created: %s" % (com.comment[:20]+"..."), request.user, incident=com.incident)
			i.refresh_artifacts(com.comment)

			if com.action.name in ['Closed', 'Opened', 'Blocked']:
				com.incident.status = com.action.name[0]
				com.incident.save()

			return render(request, 'events/_comment.html', {'event': i, 'comment': com})
		else:
			ret = {'status': 'error', 'errors': comment_form.errors}
			return HttpResponseServerError(dumps(ret), content_type="application/json")

	return redirect('incidents:details', incident_id=incident_id)


# Artifacts ===================================================================

# incidents containing a given artifact
@login_required
@user_passes_test(is_incident_handler)
def artifacts_incidents(request, artifact_id):
	a = get_object_or_404(Artifact, pk=artifact_id)
	incidents = a.incidents.all()
	return render(request, 'artifacts/incidents_index.html', {'incident_list': incidents, 'artifact': a})


@login_required
@user_passes_test(is_incident_handler)
def detach_artifact(request, artifact_id, incident_id):
	a = get_object_or_404(Artifact, pk=artifact_id)
	i = get_object_or_404(Incident, pk=incident_id)
	a.incidents.remove(i)
	if a.incidents.count() == 0:
		a.delete()
	return redirect('incidents:details', incident_id=incident_id)


# incidents related to another incident by its artifacts
@login_required
@user_passes_test(is_incident_handler)
def related_incidents(request, incident_id):
	pass


# User ==========================================================================

@login_required
@user_passes_test(is_incident_handler)
def toggle_closed(request):
	request.user.profile.hide_closed = not request.user.profile.hide_closed
	request.user.profile.save()
	response = {'status': 'ok', 'hide_closed': request.user.profile.hide_closed}
	return HttpResponse(dumps(response), content_type="application/json")


# Statistics ====================================================================

def delete_empty_keys(chart_data):
	to_delete = []

	for key in list(chart_data[0].keys()):
		delete = True
		for i in range(len(chart_data)):
			if chart_data[i].get(key) != 0:
				delete = False
		if delete:
			to_delete.append(key)

	for key in to_delete:
		for d in chart_data:
			del d[key]

	return chart_data


@login_required
@user_passes_test(can_view_statistics)
def yearly_stats(request):
	return render(request, 'stats/yearly.html')


@login_required
@user_passes_test(is_incident_handler)
def close_old(request):
	now = datetime.datetime.now()
	old = Incident.objects.filter(date__lt=datetime.datetime(now.year, now.month, 1) - datetime.timedelta(days=90))
	for i in old:
		if i.status != "C":
			i.close_timeout()

	return redirect('stats:quarterly_bl_stats')


@login_required
@user_passes_test(can_view_statistics)
def quarterly_bl_stats(request, business_line=None, num_months=3):
	bls = BusinessLine.get_parents()
	try:
		bl = BusinessLine.objects.get(name=business_line)
	except:
		bl = BusinessLine.objects.get(name=bls[0].name)

	today = datetime.date.today()
	q_month = Q()
	for i in xrange(num_months):
		previous = today-relativedelta(months=i+1)
		month = previous.month
		year = previous.year

		q_month |= Q(date__month=month, date__year=year)

	q = q_month & Q(confidentiality__lte=2)
	qbl = (Q(concerned_business_lines=bl) | Q(main_business_lines=bl) | Q(concerned_business_lines__in=bl.get_children()))
	q &= qbl

	incident_list = [i for i in Incident.objects.filter(q).order_by('-date').distinct()]
	unclosed = qbl & Q(date__lt=today - relativedelta(months=3)) & ~Q(status='C')
	unclosed_incident_list = [i for i in Incident.objects.filter(unclosed).order_by('-date').distinct()]

	return render(request, 'stats/quarterly.html', {'bl': bl, 'incident_list': incident_list, 'unclosed_incident_list': unclosed_incident_list, 'bls': bls})


@login_required
@user_passes_test(can_view_statistics)
def yearly_compare(request, year=None):
	if year is None:
		year = datetime.datetime.now().year - 1

	return render(request, 'stats/yearly_compare.html', {'year': year})


# comparison ===============================================================

@login_required
@user_passes_test(is_incident_handler)
def data_sandbox(request):

	# parse request parameters from GET requests

	category_selection_ids = [int(c) for c in request.GET.getlist('category_selection')]

	if category_selection_ids == []:
		category_selection = IncidentCategory.objects.all()
	else:
		category_selection = IncidentCategory.objects.filter(id__in=category_selection_ids)

	start = parse(request.GET['from_date'])
	end = parse(request.GET['to_date'])
	divisor = request.GET['divisor']
	graph_type = request.GET['graph_type']

	dates = []
	delta = relativedelta(end, start)
	months = delta.years*12 + delta.months

	for i in xrange(months):
			dates.append(end-relativedelta(months=i+1))

	q_categories = Q()
	for c in category_selection:
		q_categories |= Q(category=c)

	bls = [int(bl) for bl in request.GET.getlist('concerned_business_lines')]
	if len(bls) > 0:
		q_bl = Q(concerned_business_lines__in=bls) | Q(main_business_lines__in=bls)
	else:
		bls = None
		q_bl = Q()

	if request.GET["detection"] != "":
		q_detection = Q(detection=int(request.GET["detection"]))
	else:
		q_detection = Q()

	if request.GET["severity"] != "":
		if request.GET['severity_comparator'] == 'lt':
			q_severity = Q(severity__lt=int(request.GET["severity"]))
		elif request.GET['severity_comparator'] == 'gt':
			q_severity = Q(severity__gt=int(request.GET["severity"]))
		else:
			q_severity = Q(severity=int(request.GET["severity"]))
	else:
		q_severity = Q()

	if request.GET.get('is_incident', False):  # only incidents
		q_is_incident = Q(is_incident=True)
	else:
		q_is_incident = Q()

	if request.GET.get('is_major', False):  # only major incidents
		q_is_major = Q(is_major=True)
	else:
		q_is_major = Q()

	q_all = q_categories & q_detection & q_severity & q_is_incident & q_is_major & q_bl

	chart_data = []

	if graph_type == 'table':
		if divisor == 'all':
			qs = Q()
			for i in xrange(months):
				qs |= Q(date__year=dates[i].year, date__month=dates[i].month)

			incidents = Incident.objects.filter(qs & q_all).distinct()
			for inc in incidents:
				plot = {}
				plot['date'] = str(inc.date)
				plot['id'] = inc.id
				plot['subject'] = inc.subject
				plot['category'] = inc.category.name
				plot['confidentiality_display'] = inc.get_confidentiality_display()
				plot['severity'] = inc.severity
				plot['business_lines_names'] = inc.get_business_lines_names()
				plot['status_display'] = inc.get_status_display()
				plot['detection'] = str(inc.detection)
				plot['actor'] = str(inc.actor)
				plot['last_comment_action'] = inc.get_last_comment().action.name
				plot['last_comment_date'] = str(inc.get_last_comment().date)
				plot['opened_by'] = str(inc.opened_by)
				plot['plan'] = str(inc.plan)
				chart_data.append(plot)

	if graph_type == 'line':

		if divisor == 'all':
			for i in xrange(months):
				plot = {}
				plot['date'] = str(dates[i].year)+"-"+str(dates[i].month)
				plot['N'] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month) & q_all).distinct().count()
				plot['N-1'] = Incident.objects.filter(Q(date__year=dates[i].year-1, date__month=dates[i].month) & q_all).distinct().count()
				chart_data.append(plot)

		if divisor == 'category':
			for i in xrange(months):
				plot = {}
				plot['date'] = str(dates[i].year)+"-"+str(dates[i].month)

				for cat in category_selection:
					plot[cat.name] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, category=cat) & q_detection & q_severity & q_is_incident & q_is_major & q_bl).distinct().count()
				chart_data.append(plot)

	if graph_type == 'bar':

		if divisor == 'months':
			for i in xrange(months):
				plot = {}
				plot['label'] = '%s-%s' % (dates[i].year, dates[i].month)
				plot['value'] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month) & q_all).distinct().count()
				plot['text'] = plot['value']
				chart_data.append(plot)

		if divisor == 'monitoring':
			for i in xrange(months):
				plot = {}
				plot['label'] = '%s-%s' % (dates[i].year, dates[i].month)
				plot['value'] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, plan__name='A') & q_all).distinct().count()
				plot['text'] = plot['value']
				chart_data.append(plot)

		if divisor == 'open':
			for i in xrange(months):
				plot = {}
				plot['label'] = '%s-%s' % (dates[i].year, dates[i].month)
				plot['value'] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, status='O') & q_all).distinct().count()
				plot['text'] = plot['value']
				chart_data.append(plot)

		if divisor == 'blocked':
			for i in xrange(months):
				plot = {}
				plot['label'] = '%s-%s' % (dates[i].year, dates[i].month)
				plot['value'] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, status='B') & q_all).distinct().count()
				plot['text'] = plot['value']
				chart_data.append(plot)

	if graph_type == 'donut' or graph_type == 'stacked':

		if divisor == 'severity':
			for i in xrange(months):
				plot = {}
				plot['entry'] = '%s-%s' % (dates[i].year, dates[i].month)
				append = False
				for severity in xrange(1, 5):
					plot['%s/4' % severity] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, severity=severity) & q_severity & q_categories & q_detection & q_is_incident & q_is_major & q_bl).distinct().count()
					if plot['%s/4' % severity] > 0:
						append = True
				if append:
					chart_data.append(plot)

		if divisor == 'subentity':
			if not bls:
				children = BusinessLine.get_parents()
			else:
				bl = BusinessLine.objects.get(id=bl)
				children = bl.get_children()

			for i in xrange(months):
				plot = {}
				for cbl in children:
					plot[cbl.name] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, concerned_business_lines=cbl) & q_categories & q_detection & q_severity & q_is_incident & q_is_major).distinct().count()

				plot['entry'] = '%s-%s' % (dates[i].year, dates[i].month)
				chart_data.append(plot)

		if divisor == 'actor':
			for i in xrange(months):
				plot = {}
				for actor in Label.objects.filter(group__name='actor'):
					plot[actor.name] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, actor=actor) & q_all).distinct().count()

				plot['entry'] = '%s-%s' % (dates[i].year, dates[i].month)
				chart_data.append(plot)

		if divisor == 'category':
			for i in xrange(months):
				plot = {}
				for category in category_selection:
					plot[category.name] = Incident.objects.filter(Q(date__year=dates[i].year, date__month=dates[i].month, category=category) & q_detection & q_severity & q_is_incident & q_is_major & q_bl).distinct().count()

				plot['entry'] = '%s-%s' % (dates[i].year, dates[i].month)
				chart_data.append(plot)

	return HttpResponse(dumps(chart_data), content_type="application/json")


@login_required
@user_passes_test(is_incident_handler)
def sandbox(request, divisor='all'):
	form = IncidentForm(request.POST)

	if request.method == 'POST':
		fr = request.POST['from_date'][:7]
		to = request.POST['to_date'][:7]
	else:
		today = datetime.datetime.today()
		fr = "%s-%s" % (today.year-1, today.month)
		to = "%s-%s" % (today.year, today.month)

	categories = IncidentCategory.objects.all()
	categories_checked = [int(c) for c in request.POST.getlist('category_selection')]
	bls = BusinessLine.objects.all()
	unclosed_incident_list = Incident.objects.filter(~Q(status='C'))

	return render(request, 'stats/sandbox.html', {	'start': fr,
													'end': to,
													'form': form,
													'categories': categories,
													'categories_checked': categories_checked,
													'bls': bls,
													'unclosed_incident_list':unclosed_incident_list,
												})


@login_required
@user_passes_test(is_incident_handler)
def stats_attributes(request):
	form = IncidentForm()
	categories = IncidentCategory.objects.all()
	attributes = ValidAttribute.objects.exclude(unit__isnull=True)

	now = datetime.datetime.now()
	one_year_ago = now - relativedelta(years=1)
	fr = one_year_ago.strftime('%Y-%m-%d %H:%M')
	to = now.strftime('%Y-%m-%d %H:%M')

	return render(request, 'stats/attributes.html', { 'start': fr,
													  'end': to,
													  'form': form,
													  'categories': categories,
													  'attributes': attributes })


def stats_attributes_filter(request):
	'''Return query filter from request'''
	# Select only checked categories
	q_categories = Q()
	category_selection_ids = [int(c) for c in request.GET.getlist('category_selection')]
	if len(category_selection_ids) > 0:
		q_categories = Q(category_id__in=category_selection_ids)

	# Between the two specified dates
	start = parse(request.GET['from_date'])
	end = parse(request.GET['to_date'])
	q_dates = Q(date__range=(start, end))

	# Only specified business lines
	q_bl = Q()
	bls = [int(bl) for bl in request.GET.getlist('concerned_business_lines')]
	if len(bls) > 0:
		q_bl = Q(concerned_business_lines__in=bls) | Q(main_business_lines__in=bls)

	# Only the ones detected by specified entity
	q_detection = Q()
	if request.GET["detection"] != "":
		q_detection = Q(detection=int(request.GET["detection"]))

	# Only of the specified severity
	q_severity = Q()
	if request.GET["severity"] != "":
		if request.GET['severity_comparator'] == 'lt':
			q_severity = Q(severity__lt=int(request.GET["severity"]))
		elif request.GET['severity_comparator'] == 'lte':
			q_severity = Q(severity__lte=int(request.GET["severity"]))
		elif request.GET['severity_comparator'] == 'gt':
			q_severity = Q(severity__gt=int(request.GET["severity"]))
		elif request.GET['severity_comparator'] == 'gte':
			q_severity = Q(severity__gte=int(request.GET["severity"]))
		else:
			q_severity = Q(severity=int(request.GET["severity"]))

	# Only incidents if needed
	q_is_incident = Q()
	if request.GET.get('is_incident', False):
		q_is_incident = Q(is_incident=True)

	# Only major incidents if needed
	q_is_major = Q()
	if request.GET.get('is_major', False):
		q_is_major = Q(is_major=True)

	return q_categories & q_dates & q_detection & q_severity & q_is_incident & q_is_major & q_bl


def stats_attributes_get_data(filter, attribute_ids):
	incidents = Incident.objects.filter(filter).order_by('date').distinct().reverse()
	incident_ids = [i.id for i in incidents]

	valid_attributes = ValidAttribute.objects.filter(pk__in=attribute_ids)
	valid_attribute_names = [a.name for a in valid_attributes]

	all_attributes = {}
	for name in valid_attribute_names:
		all_attributes[name] = Attribute.objects.filter(incident_id__in=incident_ids, name=name).distinct()

	return (incidents, all_attributes)


def stats_attributes_date_ranges(start, end):
	start = start.replace(tzinfo=None)
	end = end.replace(tzinfo=None)
	delta = end - start

	result = []
	# Less than 3 days, use hours
	if delta.days < 3:
		for i in xrange(int(delta.total_seconds()) / 3600 + 1):
			fr = start + relativedelta(hours=i, minute=0, second=0, microsecond=0)
			to = fr + relativedelta(hours=1)
			result.append({
				"label": fr.strftime('%m-%d %H:%M'),
				"from": fr,
				"to": to,
				"x": (fr - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
			})
	# Between 3 days and 1 month, use days
	elif delta.days < 31:
		for i in xrange(int(delta.total_seconds()) / 86400 + 1):
			fr = start + relativedelta(days=i, hour=0, minute=0, second=0, microsecond=0)
			to = fr + relativedelta(days=1)
			result.append({
				"label": fr.strftime('%m-%d'),
				"from": fr,
				"to": to,
				"x": (fr - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
			})
	# Between 1 month a 10 months, use weeks
	elif delta.days < 305:
		for i in xrange(int(delta.total_seconds()) / 604800 + 1):
			fr = start + relativedelta(weeks=i, weekday=MO(-1), hour=0, minute=0, second=0, microsecond=0)
			to = fr + relativedelta(days=7)
			result.append({
				"label": str(fr.isocalendar()[1]),
				"from": fr,
				"to": to,
				"x": (fr - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
			})
	# Between 10 months and 5 years, use months
	elif delta.days < 1825:
		for i in xrange(int(delta.total_seconds()) / 2592000 + 1):
			fr = start + relativedelta(months=i, day=1, hour=0, minute=0, second=0, microsecond=0)
			to = fr + relativedelta(months=1)
			result.append({
				"label": fr.strftime('%m/%y'),
				"from": fr,
				"to": to,
				"x": (fr - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
			})
	# Otherwise, use years
	else:
		for i in xrange(int(delta.total_seconds()) / 31536000 + 1):
			fr = start + relativedelta(years=i, month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
			to = fr + relativedelta(years=1)
			result.append({
				"label": fr.strftime('%Y'),
				"from": fr,
				"to": to,
				"x": (fr - datetime.datetime(1970, 1, 1)).total_seconds() * 1000
			})

	return result


@login_required
@user_passes_test(can_view_statistics)
def stats_attributes_table(request):
	main_filter = stats_attributes_filter(request)

	result = []
	attribute_selection = request.GET.getlist("attribute_selection")
	if request.GET['bars'] != '0':
		attribute_selection.append(request.GET['bars'])
	(incidents, all_attributes) = stats_attributes_get_data(main_filter, attribute_selection)

	for incident in incidents:
		attribute_count = 0
		attributes = {}
		for valid_attribute in all_attributes:
			attributes[valid_attribute] = []
			for attribute in all_attributes[valid_attribute]:
				if attribute.incident_id == incident.id:
					attribute_count += 1
					attributes[valid_attribute].append(attribute.value)
			attributes[valid_attribute] = ', '.join(attributes[valid_attribute])

		if not request.GET.get('only_with_attribute', False) or attribute_count > 0:
			values = {}
			values['date'] = str(incident.date)
			values['category'] = incident.category.name
			values['subject'] = incident.subject
			values['business_lines_names'] = incident.get_business_lines_names()

			if incident.is_incident:
				values['url'] = reverse('incidents:details', args=[incident.id])
			else:
				values['url'] = reverse('events:details', args=[incident.id])

			values['attributes'] = attributes

			result.append(values)

	return HttpResponse(dumps(result), content_type="application/json")


def stats_attributes_by_time_range(request):
	main_filter = stats_attributes_filter(request)
	attribute_selection = request.GET.getlist("attribute_selection")
	if request.GET['bars'] != '0':
		attribute_selection.append(request.GET['bars'])
	(incidents, all_attributes) = stats_attributes_get_data(main_filter, attribute_selection)
	datetime_ranges = stats_attributes_date_ranges(parse(request.GET['from_date']), parse(request.GET['to_date']))

	for i, datetime_range in enumerate(datetime_ranges):
		# Get incident ids for selected time range
		incident_ids = []
		for incident in incidents:
			if datetime_range['from'] <= incident.date.replace(tzinfo=None) < datetime_range['to']:
				incident_ids.append(incident.id)

		with_attribute = []
		attributes = {}
		for valid_attribute in all_attributes:
			attributes[valid_attribute] = []
			for attribute in all_attributes[valid_attribute]:
				if attribute.incident_id in incident_ids:
					if attribute.incident_id not in with_attribute:
						with_attribute.append(attribute.incident_id)
					attributes[valid_attribute].append(attribute.value)

		yield (datetime_range, incident_ids, attributes, len(with_attribute))


@login_required
@user_passes_test(can_view_statistics)
def stats_attributes_over_time(request):
	incident_values = []
	attribute_values = {}
	i = 0

	bars_attribute = None
	if request.GET['bars'] != '0':
		bars_attribute = ValidAttribute.objects.get(pk=int(request.GET['bars']))

	for date_range, incidents, attributes, with_attribute_count in stats_attributes_by_time_range(request):
		# First, init dictionnary values (only the first time)
		if i == 0:
			for valid_attribute in attributes:
				attribute_values[valid_attribute] = {}
				if bars_attribute is not None and bars_attribute.name == valid_attribute:
					attribute_values[valid_attribute]['total'] = []
				else:
					if request.GET.get('total', False):
						attribute_values[valid_attribute]['total'] = []
					if request.GET.get('average', False):
						attribute_values[valid_attribute]['avg'] = []
					if request.GET.get('deviation', False):
						attribute_values[valid_attribute]['std'] = []

		# Compute bars_attribute, if any
		if bars_attribute is not None:
			attribute_values[bars_attribute.name]['total'].append(copy.deepcopy(date_range))
			attribute_values[bars_attribute.name]['total'][i]['y'] = sum(int(x) for x in attributes[bars_attribute.name])
		# Otherwise, compute incidents
		else:
			incident_values.append(copy.deepcopy(date_range))
			if request.GET.get('only_with_attribute', False):
				incident_values[i]['y'] = with_attribute_count
			else:
				incident_values[i]['y'] = len(incidents)

		# Compute all other attributes
		for valid_attribute in attributes:
			if bars_attribute is None or bars_attribute.name != valid_attribute:
				total = sum(int(x) for x in attributes[valid_attribute])
				if request.GET.get('total', False):
					attribute_values[valid_attribute]['total'].append(copy.deepcopy(date_range))
					attribute_values[valid_attribute]['total'][i]['y'] = total
				if request.GET.get('average', False):
					attribute_values[valid_attribute]['avg'].append(copy.deepcopy(date_range))
					if bars_attribute is None:
						if request.GET.get('only_with_attribute', False):
							attribute_values[valid_attribute]['avg'][i]['y'] = average([total], with_attribute_count)
						else:
							attribute_values[valid_attribute]['avg'][i]['y'] = average([total], len(incidents))
					else:
						attribute_values[valid_attribute]['avg'][i]['y'] = average([total], attribute_values[bars_attribute.name]['total'][i]['y'])
				if request.GET.get('deviation', False):
					attribute_values[valid_attribute]['std'].append(copy.deepcopy(date_range))
					attribute_values[valid_attribute]['avg'][i]['y'] = 0
					if request.GET.get('average', False):
						deviation = map(lambda x: (int(x) - attribute_values[valid_attribute]['avg'][i]['y'])**2, attributes[valid_attribute])
						attribute_values[valid_attribute]['std'][i]['y'] = math.sqrt(average(deviation, len(deviation)))

		i = i + 1

	result = []

	# Add bars attribute, if any
	if bars_attribute is not None:
		result.append({"key": bars_attribute.name, "values": attribute_values[bars_attribute.name]['total'], "bar": True})
	# Otherwise, take incidents as bars
	else:
		result.append({"key": "Incidents", "values": incident_values, "bar": True})

	# Add every attribute
	for attribute in attribute_values:
		if bars_attribute is None or bars_attribute.name != attribute:
			for stat in attribute_values[attribute]:
				result.append({"key": "%s %s" % (stat, attribute), "values": attribute_values[attribute][stat]})

	return HttpResponse(dumps(result, default=json_util.default), content_type="application/json")


def average(values, size):
	if size > 0:
		return sum(values) / size
	else:
		return 0


@login_required
@user_passes_test(can_view_statistics)
def stats_attributes_basic(request):
	main_filter = stats_attributes_filter(request)

	attribute_selection = request.GET.getlist("attribute_selection")

	bars_attribute = None
	if request.GET['bars'] != '0':
		attribute_selection.append(request.GET['bars'])
		bars_attribute = ValidAttribute.objects.get(pk=int(request.GET['bars']))

	(incidents, all_attributes) = stats_attributes_get_data(main_filter, attribute_selection)

	incidents_with_attribute = []
	for valid_attribute in all_attributes:
		for attribute in all_attributes[valid_attribute]:
			if attribute.incident_id not in incidents_with_attribute:
				incidents_with_attribute.append(attribute.incident_id)

	attributes = {}
	if bars_attribute is not None:
		attributes[bars_attribute.name] = {}
		attributes[bars_attribute.name]['total'] = sum(int(x.value) for x in all_attributes[bars_attribute.name])

	for valid_attribute in all_attributes:
		if bars_attribute is None or bars_attribute.name != valid_attribute:
			attributes[valid_attribute] = {}
			total = sum(int(x.value) for x in all_attributes[valid_attribute])
			if request.GET.get('total', False):
				attributes[valid_attribute]['total'] = total
			if request.GET.get('average', False):
				attributes[valid_attribute]['average'] = 0
				if bars_attribute is None:
					if request.GET.get('only_with_attribute', False):
						attributes[valid_attribute]['average'] = average([total], len(incidents_with_attribute))
					else:
						attributes[valid_attribute]['average'] = average([total], len(incidents))
				else:
					attributes[valid_attribute]['average'] = average([total], attributes[bars_attribute.name]['total'])
			if request.GET.get('deviation', False):
				attributes[valid_attribute]['standard deviation'] = 0
				if request.GET.get('average', False):
					deviation = map(lambda x: (int(x.value) - attributes[valid_attribute]['average'])**2, all_attributes[valid_attribute])
					attributes[valid_attribute]['standard deviation'] = math.sqrt(average(deviation, len(deviation)))

	result = {
		"incidents": len(incidents),
		"with_attribute": len(incidents_with_attribute),
		"attributes": attributes
	}

	return HttpResponse(dumps(result, default=json_util.default), content_type="application/json")


@login_required
@user_passes_test(can_view_statistics)
# set slide=True for a sliding year (current month, 12 months backwards).
# slide=False for comparing Y-jan -> Y-dec and (Y-1)-jan -> (Y-1)-dec
def data_yearly_compare(request, year, type='all', slide=True):
	chart_data = []

	year = int(year)

	q = Q()

	if type == 'incidents':
			q = Q(is_incident=True)
	elif type == 'events':
			q = Q(is_incident=False)

	if slide:
		dates = []
		today = datetime.date.today().replace(year=year)
		for i in xrange(12):
			dates.append(today-relativedelta(months=i+1))

		for i in xrange(12):
			chart_data.append({	'date': str(dates[i].year) + "-" + str(dates[i].month),
								str(year): Incident.objects.filter(q).filter(date__year=dates[i].year, date__month=dates[i].month).count(),
								str(year-1): Incident.objects.filter(q).filter(date__year=dates[i].year-1, date__month=dates[i].month).count()
								})

	else:
		for i in xrange(12):
			chart_data.append({	'date': str(year)+"-"+str(i+1),
								str(year): Incident.objects.filter(q).filter(date__year=year, date__month=i+1).count(),
								str(year-1):  Incident.objects.filter(q).filter(date__year=year-1, date__month=i+1).count()
								})

	return HttpResponse(dumps(chart_data), content_type="application/json")


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_evolution(request, year, type='all', divisor='bl', slide=True):
	chart_data = []

	year = int(year)

	q = Q()

	if type == 'incidents':
		q = Q(is_incident=True)
	elif type == 'events':
		q = Q(is_incident=False)

	if divisor == 'bl':
		items = BusinessLine.get_parents()
	elif divisor == 'category':
		items = IncidentCategory.objects.all()

	for item in items:
		d = {}
		if divisor == 'bl':
			new_q = q & (Q(concerned_business_lines=item) | Q(main_business_lines=item) | Q(concerned_business_lines__in=item.get_children()))
		elif divisor == 'category':
			new_q = q & Q(category=item)

		d['label'] = item.name
		if slide:
			today = datetime.datetime.now().replace(day=1, year=year)
			dates = Q()
			for i in xrange(12):
				dates |= Q(date__month=(today-relativedelta(months=i)).month, date__year=(today-relativedelta(months=i)).year)
			d['value'] = Incident.objects.filter(new_q & dates).distinct().count()
			dates = Q()
			for i in xrange(12):
				dates |= Q(date__month=(today-relativedelta(months=i)).month, date__year=(today-relativedelta(months=i)).year-1)

			previous_value = Incident.objects.filter(new_q & dates).distinct().count()
		else:
			d['value'] = Incident.objects.filter(new_q & Q(date__year=year)).distinct().count()
			previous_value = Incident.objects.filter(new_q & Q(date__year=year-1)).distinct().count()

		if previous_value > 0:
			delta = d['value'] - previous_value
			delta = delta/float(previous_value)
			delta = delta*100
			delta = int(round(delta))
			if delta > 0:
				d['text'] = "+"+str(delta)+"%"
			else:
				d['text'] = str(delta)+"%"
		else:
			d['text'] = "new!"

		d['text'] = str(d['text'])
		if d['value'] != 0 or previous_value != 0:
			chart_data.append(d)

	return HttpResponse(dumps(chart_data), content_type="application/json")


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_incidents(request):

	chart_data = []

	dates = []
	today = datetime.date.today()
	for i in xrange(12):
		dates.append(today-relativedelta(months=i+1))
	for i in xrange(12):
		chart_data.append({'date': str(dates[i].year)+"-"+str(dates[i].month), today.year-1: Incident.objects.filter(date__year=dates[i].year, date__month=dates[i].month).count()})

	return HttpResponse(dumps(chart_data), content_type="application/json")


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_bl(request, year=datetime.date.today().year, type='all'):
	bls = BusinessLine.get_parents()

	chart_data = []
	total = 0

	q = Q(date__year=year)

	if type == 'incidents':
		q = q & Q(is_incident=True)
	elif type == 'events':
		q = q & Q(is_incident=False)

	for bl in bls:
		d = {}
		d['label'] = bl.name
		d['value'] = bl.get_incident_count(q)
		if d['value'] > 0:
			chart_data.append(d)
		total += d['value']

	for d in chart_data:

		d['percentage'] = float(str(round(float(d['value'])/total, 2)*100))

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type="application/json")


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_bl_detection(request):
	bls = BusinessLine.get_parents()
	q = Q(is_incident=True, date__year=datetime.datetime.now().year)
	chart_data = []

	for bl in bls:
		d = {}
		d['entry'] = bl.name
		cert = bl.get_incident_count(q & Q(detection__name='CERT'))
		ext = bl.get_incident_count(q & Q(detection__name='External'))
		d['CERT'] = cert
		d['External'] = ext
		if d['CERT'] != 0 or d['External'] != 0:
			chart_data.append(d)

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type='application/json')
	pass


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_bl_severity(request):
	bls = BusinessLine.get_parents()

	q = Q(is_incident=True, date__year=datetime.datetime.now().year)
	chart_data = []

	for bl in bls:
		d = {}
		d['entry'] = bl.name

		append = False
		for i in xrange(4):
			d[str(i+1)+'/4'] = bl.get_incident_count(q & Q(severity=i+1))
			if d[str(i+1)+'/4'] > 0:
				append = True

		if append:
			chart_data.append(d)

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type='application/json')
	pass


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_bl_category(request):
	bls = BusinessLine.get_parents()
	categories = IncidentCategory.objects.all()

	q = Q(is_incident=True, date__year=datetime.datetime.now().year)
	chart_data = []

	for bl in bls:

		d = {}
		d['entry'] = bl.name

		append = False
		for c in categories:
			d[c.name] = bl.get_incident_count(q & Q(category=c))
			if d[c.name] > 0:
				append = True

		if append:
			chart_data.append(d)

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type='application/json')
	pass


@login_required
@user_passes_test(can_view_statistics)
def data_yearly_bl_plan(request):
	bls = BusinessLine.get_parents()
	plans = Label.objects.filter(group__name='plan')

	q = Q(is_incident=True, date__year=datetime.datetime.now().year)
	chart_data = []

	for bl in bls:

		d = {}
		d['entry'] = bl.name

		append = False
		for p in plans:
			d[p.name] = bl.get_incident_count(q & Q(plan=p))
			if d[p.name] > 0:
				append = True

		if append:
			chart_data.append(d)

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type='application/json')


@login_required
@user_passes_test(can_view_statistics)
def data_incident_variation(request, business_line, num_months=3):
	bl = get_object_or_404(BusinessLine, name=business_line)

	categories = IncidentCategory.objects.all()

	chart_data = []

	q = Q(concerned_business_lines=bl) | Q(main_business_lines=bl) | Q(concerned_business_lines__in=bl.get_children())
	q &= Q(is_incident=True)
	total = 0
	total_previous = 0
	for cat in categories:

		d = {}
		current = datetime.datetime.today() - relativedelta(months=1)

		d['category'] = cat.name
		current_month = q & Q(date__month=current.month, date__year=current.year, category=cat)

		d['values'] = {}
		d['values']['new'] = Incident.objects.filter(current_month).distinct().count()
		total += d['values']['new']

		previous = current - relativedelta(months=1)

		previous_month = q & Q(date__month=previous.month, date__year=previous.year, category=cat)
		previous_count = Incident.objects.filter(previous_month).distinct().count()
		d['values']['variation'] = d['values']['new'] - previous_count

		total_previous += previous_count

		if d['values']['variation'] > 0:
			d['values']['variation'] = "+"+str(d['values']['variation'])
		else:
			d['values']['variation'] = str(d['values']['variation'])

		if d['values']['variation'] != "0" or d['values']['new'] != 0:
			chart_data.append(d)

	if total-total_previous > 0:
		total_previous = "+"+str(total-total_previous)
	else:
		total_previous = str(total-total_previous)

	chart_data.append({'category': 'Total',  'values': {'new': total, 'variation': total_previous}})

	return HttpResponse(dumps(chart_data), content_type='application/json')


@login_required
@user_passes_test(can_view_statistics)
def data_quarterly_bl(request, business_line, divisor, num_months=3, is_incident=True):
	bl = get_object_or_404(BusinessLine, name=business_line)
	children = bl.get_children()

	q = Q(main_business_lines=bl) | Q(concerned_business_lines=bl) | Q(concerned_business_lines__in=children)
	q = q & Q(confidentiality__lte=2)

	chart_data = []

	if divisor == 'incidents':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['label'] = cal[today.month-1]
			date_q = q & Q(date__month=today.month, date__year=today.year)

			d['value'] = Incident.objects.filter(date_q).distinct().count()
			d['text'] = d['value']

			chart_data.append(d)

	if divisor == 'severity':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['entry'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			d['1/4'] = Incident.objects.filter(Q(severity=1) & q_date).distinct().count()
			d['2/4'] = Incident.objects.filter(Q(severity=2) & q_date).distinct().count()
			d['3/4'] = Incident.objects.filter(Q(severity=3) & q_date).distinct().count()
			d['4/4'] = Incident.objects.filter(Q(severity=4) & q_date).distinct().count()

			chart_data.append(d)

	if divisor == 'category':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['entry'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			for cat in IncidentCategory.objects.all():
				d[cat.name] = Incident.objects.filter(q_date & Q(category=cat)).distinct().count()

			chart_data.append(d)

	if divisor == 'entity':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['entry'] = cal[today.month-1]

			q_date = q & Q(date__month=today.month, date__year=today.year)
			d[bl.name] = Incident.objects.filter(q_date).distinct().count()

			for entity in BusinessLine.objects.filter(parent=bl).distinct():
				d[entity.name] = entity.get_incident_count(q_date)
				d[bl.name] -= d[entity.name]

			chart_data.append(d)

	if divisor == 'actor':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['entry'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			for actor in Label.objects.filter(group__name='actor').distinct():
				d[actor.name] = Incident.objects.filter(q_date & Q(actor=actor)).distinct().count()

			chart_data.append(d)

	if divisor == 'monitoring':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['label'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			d['value'] = Incident.objects.filter(q_date & Q(plan__name='A')).distinct().count()
			d['text'] = d['value']

			chart_data.append(d)

	if divisor == 'open':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['label'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			d['value'] = Incident.objects.filter(q_date & Q(status='O')).distinct().count()
			d['text'] = d['value']
			chart_data.append(d)

	if divisor == 'blocked':
		for i in xrange(num_months):
			d = {}

			today = datetime.datetime.today() - relativedelta(months=num_months-i)
			d['label'] = cal[today.month-1]
			q_date = q & Q(date__month=today.month, date__year=today.year)

			d['value'] = Incident.objects.filter(q_date & Q(status='B')).distinct().count()
			d['text'] = d['value']
			chart_data.append(d)

	chart_data = delete_empty_keys(chart_data)

	return HttpResponse(dumps(chart_data), content_type='application/json')


@login_required
@user_passes_test(can_view_statistics)
def quarterly_major(request, start_date=None, num_months=3):

	q_major = Q(is_major=True, is_incident=True)
	balecats = BaleCategory.objects.filter(Q(parent_category__isnull=False))
	certcats = IncidentCategory.objects.all()
	parent_bls = BusinessLine.get_parents()

	num_months = int(num_months)

	bale = []
	cert = []
	bls = []

	if start_date is None:
		today = datetime.datetime.today()
	else:
		today = datetime.datetime.strptime(start_date, "%Y-%m-%d")

	cert.append(['Category'])
	for i in xrange(num_months):
		then = today - relativedelta(months=num_months-i)
		cert[0].append(cal[then.month-1])
	cert[0].append('Total')
	for certcat in certcats:
		line = []
		q_certcat = Q(category=certcat) & q_major
		line.append(certcat.name)
		add = False
		total = 0
		for i in xrange(num_months):
			then = today - relativedelta(months=num_months-i)
			q_date = q_certcat & Q(date__month=then.month, date__year=then.year)
			count = Incident.objects.filter(q_date).count()
			line.append(count)

			if count != 0:
				add = True
				total += count

		line.append(total)

		if add:
			cert.append(line)

	bale.append(['Bale category'])
	for i in xrange(num_months):
		then = today - relativedelta(months=num_months-i)
		bale[0].append(cal[then.month-1])

	for balecat in balecats:
		line = []
		q_balecat = Q(category__bale_subcategory=balecat) & q_major
		line.append(balecat.__unicode__)
		add = False
		for i in xrange(num_months):
			then = today - relativedelta(months=num_months-i)
			q_date = q_balecat & Q(date__month=then.month, date__year=then.year)
			count = Incident.objects.filter(q_date).count()
			line.append(count)
			if count != 0:
				add = True

		if add:
			bale.append(line)

	bls.append(['Business Line'])
	for i in xrange(num_months):
		then = today - relativedelta(months=num_months-i)
		bls[0].append(cal[then.month-1])
	bls[0].append('Total')

	for bl in parent_bls:
		line = []
		q_bl = Q(main_business_lines=bl) & q_major
		line.append(bl.__unicode__)
		add = False

		total = 0
		for i in xrange(num_months):
			then = today - relativedelta(months=num_months-i)
			q_date = q_bl & Q(date__month=then.month, date__year=then.year)

			count = Incident.objects.filter(q_date).count()
			line.append(count)
			if count != 0:
				add = True
				total += count
		line.append(total)
		if add:
			bls.append(line)

	total_major = 0
	for i in xrange(num_months):
		d = today - relativedelta(months=num_months-i)
		total_major += Incident.objects.filter(date__month=d.month, date__year=d.year).count()

	past_months = Q()
	for i in xrange(num_months):
		d = today - relativedelta(months=num_months-i)
		past_months |= Q(date__month=d.month, date__year=d.year)

	return render(request, 'stats/major.html', {'bale': bale, 'cert': cert, 'total_major': total_major, 'bls': bls, 'incident_list': Incident.objects.filter(q_major & past_months).order_by('-date')})


# Dashboard =======================================================

def incidents_order(request):
	order_param = request.GET.get('order_by', 'date')
	asc = request.GET.get('asc', 'false')
	order_by = order_param

	if order_param == 'last_action':
		order_by = 'comments__date__max'

	if asc == 'false':
		order_by = "-%s" % order_by

	return (order_by, order_param, asc)


def incident_display(request, filter, incident_view=True, paginated=True):
	(order_by, order_param, asc) = incidents_order(request)

	if order_param == 'last_action':
		incident_list = Incident.objects.filter(filter).annotate(Max('comments__date')).order_by(order_by)
	else:
		incident_list = Incident.objects.filter(filter).order_by(order_by)

	if paginated:
		page = request.GET.get('page', 1)
		incidents_per_page = request.user.profile.incident_number
		p = Paginator(incident_list, incidents_per_page)

		try:
			incident_list = p.page(page)
		except (PageNotAnInteger, EmptyPage):
			incident_list = p.page(1)

	return render(request, 'events/table.html', {
			'incident_list': incident_list,
			'incident_view': incident_view,
			'order_param': order_param,
			'asc': asc
		})


@login_required
@user_passes_test(is_incident_handler)
def dashboard_main(request):
	return render(request, 'dashboard/index.html')


@login_required
@user_passes_test(is_incident_handler)
def dashboard_starred(request):
	return incident_display(request, Q(is_starred=True) & ~Q(status='C'), True, False)


@login_required
@user_passes_test(is_incident_handler)
def dashboard_open(request):
	return incident_display(request, Q(status='O'))


@login_required
@user_passes_test(is_incident_handler)
def dashboard_blocked(request):
	return incident_display(request, Q(status='B'))


@login_required
@user_passes_test(is_incident_handler)
def dashboard_old(request):
	incident_list = Incident.objects.filter(status='O').annotate(Max('comments__date')).order_by('comments__date__max')[:20]

	return render(request, 'events/table.html', {
			'incident_list': incident_list,
			'incident_view': True,
			'order_param': 'last_action',
			'asc': 'true'
		})
