# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals, print_function

from six.moves import range
import requests
import frappe, json, os
import frappe.permissions
import frappe.async

from frappe import _

from frappe.utils.csvutils import getlink
from frappe.utils.dateutils import parse_date
from frappe.utils.file_manager import save_url

from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_url_to_form
from six import text_type, string_types


@frappe.whitelist()
def get_data_keys():
	return frappe._dict({
		"data_separator": _('Start entering data below this line'),
		"main_table": _("Table") + ":",
		"parent_table": _("Parent Table") + ":",
		"columns": _("Column Name") + ":",
		"doctype": _("DocType") + ":"
	})



@frappe.whitelist()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
	skip_errors = True, data_import_doc=None, validate_template=False, user=None):
	"""upload data"""

	# for translations
	if user:
		frappe.set_user_lang(user)

	if data_import_doc and isinstance(data_import_doc, string_types):
		data_import_doc = frappe.get_doc("Data Import", data_import_doc)
	if data_import_doc and from_data_import == "Yes":
		no_email = data_import_doc.no_email
		ignore_encoding_errors = data_import_doc.ignore_encoding_errors
		update_only = data_import_doc.only_update
		submit_after_import = data_import_doc.submit_after_import
		overwrite = data_import_doc.overwrite
		skip_errors = data_import_doc.skip_errors
	else:
		# extra input params
		params = json.loads(frappe.form_dict.get("params") or '{}')
		if params.get("submit_after_import"):
			submit_after_import = True
		if params.get("ignore_encoding_errors"):
			ignore_encoding_errors = True
		if not params.get("no_email"):
			no_email = False
		if params.get('update_only'):
			update_only = True
		if params.get('from_data_import'):
			from_data_import = params.get('from_data_import')
		if not params.get('skip_errors'):
			skip_errors = params.get('skip_errors')

	frappe.flags.in_import = True
	frappe.flags.mute_emails = no_email

	def get_data_keys_definition():
		return get_data_keys()

	def bad_template():
		frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))

	def check_data_length():
		if not data:
			frappe.throw(_("No data found in the file. Please reattach the new file with data."))

	def get_start_row():
		for i, row in enumerate(rows):
			if row and row[0]==get_data_keys_definition().data_separator:
				return i+1
		bad_template()

	def get_header_row(key):
		return get_header_row_and_idx(key)[0]

	def get_header_row_and_idx(key):
		for i, row in enumerate(header):
			if row and row[0]==key:
				return row, i
		return [], -1

	def filter_empty_columns(columns):
		empty_cols = list(filter(lambda x: x in ("", None), columns))

		if empty_cols:
			if columns[-1*len(empty_cols):] == empty_cols:
				# filter empty columns if they exist at the end
				columns = columns[:-1*len(empty_cols)]
			else:
				frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
					raise_exception=1)

		return columns

	def make_column_map():
		doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
		if row_idx == -1: # old style
			return

		dt = None
		for i, d in enumerate(doctype_row[1:]):
			if d not in ("~", "-"):
				if d and doctype_row[i] in (None, '' ,'~', '-', 'DocType:'):
					dt, parentfield = d, None
					# xls format truncates the row, so it may not have more columns
					if len(doctype_row) > i+2:
						parentfield = doctype_row[i+2]
					doctypes.append((dt, parentfield))
					column_idx_to_fieldname[(dt, parentfield)] = {}
					column_idx_to_fieldtype[(dt, parentfield)] = {}
				if dt:
					column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
					column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]

	def get_doc(start_idx):
		if doctypes:
			doc = {}
			attachments = []
			last_error_row_idx = None
			for idx in range(start_idx, len(rows)):
				last_error_row_idx = idx	# pylint: disable=W0612
				if (not doc) or main_doc_empty(rows[idx]):
					for dt, parentfield in doctypes:
						d = {}
						for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
							try:
								fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
								fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]

								if not fieldname or not rows[idx][column_idx]:
									continue

								d[fieldname] = rows[idx][column_idx]
								if fieldtype in ("Int", "Check"):
									d[fieldname] = cint(d[fieldname])
								elif fieldtype in ("Float", "Currency", "Percent"):
									d[fieldname] = flt(d[fieldname])
								elif fieldtype == "Date":
									if d[fieldname] and isinstance(d[fieldname], string_types):
										d[fieldname] = getdate(parse_date(d[fieldname]))
								elif fieldtype == "Datetime":
									if d[fieldname]:
										if " " in d[fieldname]:
											_date, _time = d[fieldname].split()
										else:
											_date, _time = d[fieldname], '00:00:00'
										_date = parse_date(d[fieldname])
										d[fieldname] = get_datetime(_date + " " + _time)
									else:
										d[fieldname] = None

								elif fieldtype in ("Image", "Attach Image", "Attach"):
									# added file to attachments list
									attachments.append(d[fieldname])

								elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]:
									# as fields can be saved in the number format(long type) in data import template
									d[fieldname] = cstr(d[fieldname])

							except IndexError:
								pass

						# scrub quotes from name and modified
						if d.get("name") and d["name"].startswith('"'):
							d["name"] = d["name"][1:-1]

						if sum([0 if not val else 1 for val in d.values()]):
							d['doctype'] = dt
							if dt == doctype:
								doc.update(d)
							else:
								if not overwrite and doc.get("name"):
									d['parent'] = doc["name"]
								d['parenttype'] = doctype
								d['parentfield'] = parentfield
								doc.setdefault(d['parentfield'], []).append(d)
				else:
					break

			return doc, attachments, last_error_row_idx
		else:
			doc = frappe._dict(zip(columns, rows[start_idx][1:]))
			doc['doctype'] = doctype
			return doc

	def main_doc_empty(row):
		return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))

	def validate_naming(doc):
		autoname = frappe.get_meta(doctype).autoname
		if autoname:
			if autoname[0:5] == 'field':
				autoname = autoname[6:]
			elif autoname == 'naming_series:':
				autoname = 'naming_series'
			else:
				return True

			if (autoname not in doc) or (not doc[autoname]):
				from frappe.model.base_document import get_controller
				if not hasattr(get_controller(doctype), "autoname"):
					frappe.throw(_("{0} is a mandatory field".format(autoname)))
		return True

	users = frappe.db.sql_list("select name from tabUser")
	def prepare_for_insert(doc):
		# don't block data import if user is not set
		# migrating from another system
		if not doc.owner in users:
			doc.owner = frappe.session.user
		if not doc.modified_by in users:
			doc.modified_by = frappe.session.user

	def is_valid_url(url):
		is_valid = False
		if url.startswith("/files") or url.startswith("/private/files"):
			url = get_url(url)

		try:
			r = requests.get(url)
			is_valid = True if r.status_code == 200 else False
		except Exception:
			pass

		return is_valid

	def attach_file_to_doc(doctype, docname, file_url):
		# check if attachment is already available
		# check if the attachement link is relative or not
		if not file_url:
			return
		if not is_valid_url(file_url):
			return

		files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
			attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
				doctype=doctype,
				docname=docname,
				file_url=file_url
			))

		if files:
			# file is already attached
			return

		save_url(file_url, None, doctype, docname, "Home/Attachments", 0)

	# header
	filename, file_extension = ['','']
	if not rows:
		from frappe.utils.file_manager import get_file # get_file_doc
		fname, fcontent = get_file(data_import_doc.import_file)
		filename, file_extension = os.path.splitext(fname)

		if file_extension == '.xlsx' and from_data_import == 'Yes':
			from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
			rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)

		elif file_extension == '.csv':
			from frappe.utils.csvutils import read_csv_content
			rows = read_csv_content(fcontent, ignore_encoding_errors)

		else:
			frappe.throw(_("Unsupported File Format"))

	start_row = get_start_row()
	header = rows[:start_row]
	data = rows[start_row:]
	doctype = get_header_row(get_data_keys_definition().main_table)[1]
	columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
	doctypes = []
	column_idx_to_fieldname = {}
	column_idx_to_fieldtype = {}

	if skip_errors:
		data_rows_with_error = header

	if submit_after_import and not cint(frappe.db.get_value("DocType",
			doctype, "is_submittable")):
		submit_after_import = False

	parenttype = get_header_row(get_data_keys_definition().parent_table)

	if len(parenttype) > 1:
		parenttype = parenttype[1]

	# check permissions
	if not frappe.permissions.can_import(parenttype or doctype):
		frappe.flags.mute_emails = False
		return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}

	# Throw expception in case of the empty data file
	check_data_length()
	make_column_map()
	total = len(data)

	if validate_template:
		if total:
			data_import_doc.total_rows = total
		return True

	if overwrite==None:
		overwrite = params.get('overwrite')

	# delete child rows (if parenttype)
	parentfield = None
	if parenttype:
		parentfield = get_parent_field(doctype, parenttype)

		if overwrite:
			delete_child_rows(data, doctype)

	import_log = []
	def log(**kwargs):
		if via_console:
			print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
		else:
			import_log.append(kwargs)

	def as_link(doctype, name):
		if via_console:
			return "{0}: {1}".format(doctype, name)
		else:
			return getlink(doctype, name)

	# publish realtime task update
	def publish_progress(achieved, reload=False):
		if data_import_doc:
			frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
				"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)


	error_flag = rollback_flag = False
	for i, row in enumerate(data):
		# bypass empty rows
		if main_doc_empty(row):
			continue

		row_idx = i + start_row
		doc = None

		publish_progress(i)

		try:
			doc, attachments, last_error_row_idx = get_doc(row_idx)
			validate_naming(doc)
			if pre_process:
				pre_process(doc)

			original = None
			if parentfield:
				parent = frappe.get_doc(parenttype, doc["parent"])
				doc = parent.append(parentfield, doc)
				parent.save()
			else:
				if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
					original = frappe.get_doc(doctype, doc["name"])
					original_name = original.name
					original.update(doc)
					# preserve original name for case sensitivity
					original.name = original_name
					original.flags.ignore_links = ignore_links
					original.save()
					doc = original
				else:
					if not update_only:
						doc = frappe.get_doc(doc)
						prepare_for_insert(doc)
						doc.flags.ignore_links = ignore_links
						doc.insert()
				if attachments:
					# check file url and create a File document
					for file_url in attachments:
						attach_file_to_doc(doc.doctype, doc.name, file_url)
				if submit_after_import:
					doc.submit()

			# log errors
			if parentfield:
				log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
					"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
			elif submit_after_import:
				log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
					"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
			elif original:
				log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
					"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
			elif not update_only:
				log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
					"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
			else:
				log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
					"message": "Document updation ignored", "indicator": "orange"})

		except Exception as e:
			error_flag = True
			err_msg = frappe.local.message_log and "\n".join([json.loads(msg).get('message') for msg in frappe.local.message_log]) or cstr(e)
			error_trace = frappe.get_traceback()
			if error_trace:
				error_log_doc = frappe.log_error(error_trace)
				error_link = get_url_to_form("Error Log", error_log_doc.name)
			else:
				error_link = None
			log(**{"row": row_idx + 1, "title":'Error for row %s' % (len(row)>1 and row[1] or ""), "message": err_msg,
				"indicator": "red", "link":error_link})
			# data with error to create a new file
			# include the errored data in the last row as last_error_row_idx will not be updated for the last row
			if skip_errors:
				if last_error_row_idx == len(rows)-1:
					last_error_row_idx = len(rows)
				data_rows_with_error += rows[row_idx:last_error_row_idx]
			else:
				rollback_flag = True
		finally:
			frappe.local.message_log = []

	if rollback_flag:
		frappe.db.rollback()
	else:
		frappe.db.commit()

	frappe.flags.mute_emails = False
	frappe.flags.in_import = False

	log_message = {"messages": import_log, "error": error_flag}
	if data_import_doc:
		data_import_doc.log_details = json.dumps(log_message)

		import_status = None
		if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
			import_status = "Partially Successful"
			# write the file with the faulty row
			from frappe.utils.file_manager import save_file
			file_name = 'error_' + filename + file_extension
			if file_extension == '.xlsx':
				from frappe.utils.xlsxutils import make_xlsx
				xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
				file_data = xlsx_file.getvalue()
			else:
				from frappe.utils.csvutils import to_csv
				file_data = to_csv(data_rows_with_error)
			error_data_file = save_file(file_name, file_data, "Data Import",
				data_import_doc.name,  "Home/Attachments")
			data_import_doc.error_file = error_data_file.file_url

		elif error_flag:
			import_status = "Failed"
		else:
			import_status = "Successful"

		data_import_doc.import_status = import_status
		data_import_doc.save()
		if data_import_doc.import_status in ["Successful", "Partially Successful"]:
			data_import_doc.submit()
			publish_progress(100, True)
		else:
			publish_progress(0, True)
		frappe.db.commit()
	else:
		return log_message

def get_parent_field(doctype, parenttype):
	parentfield = None

	# get parentfield
	if parenttype:
		for d in frappe.get_meta(parenttype).get_table_fields():
			if d.options==doctype:
				parentfield = d.fieldname
				break

		if not parentfield:
			frappe.msgprint(_("Did not find {0} for {0} ({1})").format("parentfield", parenttype, doctype))
			raise Exception

	return parentfield

def delete_child_rows(rows, doctype):
	"""delete child rows for all parents"""
	for p in list(set([r[1] for r in rows])):
		if p:
			frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p)
