import json
import os
import urllib
import time
import glob

import webapp2
import jinja2
from google.appengine.api import users
from google.appengine.ext import db
from au.data.access_control import may_see_managedata_view
from au.data.datastore import DocumentAnnotation, ArgumentationUnit, Document, \
    UserData, CorpusMetadata
from google.appengine.api.datastore_types import Text
from au.data.utils import initialize_document, extract_metadata
from au.data import access_control
from au.data.processing_state import UNPROCESSED, IN_PROGRESS, COMPLETE

__author__ = "Artem Vovk, Roland Kluge, and Christian Kirschner"
__copyright__ = "Copyright 2013-2015 UKP TU Darmstadt"
__credits__ = ["Artem Vovk", "Roland Kluge", "Christian Kirschner"]
__license__ = "ASL"

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__) + "/templates/"),
    extensions=['jinja2.ext.autoescape'])

DEFAULT_CONFIDENCE = "\"high\""


class ManageDataHandler(webapp2.RequestHandler):
    """
    Allows to manage data (load, delete, update)
    
    @author: Roland Kluge
    """

    def validate_user(self):
        user_id = access_control.get_current_user_id()

        if not user_id:
            self.redirect(users.create_login_url(self.request.uri))
            return False

        if not may_see_managedata_view(user_id):
            self.response.write(
                'User ' + user_id + ' is not allowed to access this page! <a href="' + users.create_logout_url(
                    '/argunit/managedata') + '">Logout</a>')
            return False

        return True

    def base_path(self):
        return '/argunit/managedata'

    def get(self):

        user_id = access_control.get_current_user_id()

        if not self.validate_user():
            return
        elif self.request.path.endswith('dump'):
            self.dump_corpus()
        elif self.request.path.endswith('dropall'):
            self.drop_all_data()
            self.redirect(
                '%s?%s' % (self.base_path(), urllib.urlencode({"message": 'Dropped all data.'})))

        elif self.request.path.endswith('dropanno'):
            self.drop_all_annotations()
            self.redirect('%s?%s' % (
            self.base_path(), urllib.urlencode({"message": 'Dropped all annotations.'})))

        elif self.request.path.endswith('loaddata'):
            response_text = self.load_documents()
            self.redirect(
                '%s?%s' % (self.base_path(), urllib.urlencode({"verbatim_message": response_text})))

        elif self.request.path.endswith('forceupdate'):
            response_text = self.load_documents(force_update=True)
            self.redirect(
                '%s?%s' % (self.base_path(), urllib.urlencode({"verbatim_message": response_text})))
        elif self.request.path.endswith('unapprove'):
            annotator = self.request.get("annotator")
            document = self.request.get("doc")
            self.setApproval(annotator, document, False);
            response_text = "Unapproved: %s:%s" % (annotator, document)
            self.redirect(
                '%s?%s' % (self.base_path(), urllib.urlencode({"message": response_text})))
        elif self.request.path.endswith('/managedata'):

            all_documents = [doc.filename for doc in Document.all()]
            all_documents.sort()
            all_users = access_control.get_all_users()
            all_users.sort()
            status_table = dict()
            for user in all_users:
                status_table[user] = dict()
                for doc in all_documents:
                    anno = DocumentAnnotation.all().filter("user_id =", user).filter("document =",
                                                                                     doc).get()
                    if not anno:
                        status_table[user][doc] = UNPROCESSED
                    elif not anno.approved:
                        status_table[user][doc] = IN_PROGRESS
                    else:
                        status_table[user][doc] = COMPLETE

            documents_per_line = 44
            num_docs = len(all_documents)
            num_lines = (num_docs + documents_per_line - 1) / documents_per_line
            partitioned_docs = []
            for i in range(0, num_lines):
                partitioned_docs.append(all_documents[i * documents_per_line:min(num_docs, (
                i + 1) * documents_per_line)])

            message = self.request.get('message', "")
            verbatim_message = self.request.get('verbatim_message', "")

            metadata = CorpusMetadata.all().get()
            segmenter = "unknown"
            preprocessing_date = "unknown"
            if metadata:
                segmenter = metadata.segmenter
                preprocessing_date = metadata.preprocessing_date

            template_values = {'user': user_id,
                               'logout_url': users.create_logout_url('/argunit/'),
                               'all_views': access_control.get_view_ids(user_id),
                               'current_view': access_control.MANAGE_DATA_VIEW_ID,
                               'num_documents': len(all_documents),
                               'segmenter': segmenter,
                               'preprocessing_date': preprocessing_date,
                               'all_documents': all_documents,
                               'docs_per_line': documents_per_line,
                               'partitioned_docs': partitioned_docs,
                               'all_users': all_users,
                               'status_table': status_table,
                               'message': message,
                               'verbatim_message': verbatim_message}
            template = JINJA_ENVIRONMENT.get_template('managedata.html')
            self.response.write(template.render(template_values))
        else:
            self.redirect('/argunit/managedata')

    def dump_corpus(self):
        docs = Document.all().run()
        jsonResponse = []
        metadata = CorpusMetadata.all().get()
        if metadata:
            jsonResponse.append({"corpus_metadata": "true",
                                 "segmenter": metadata.segmenter,
                                 "preprocessing_date": metadata.preprocessing_date
                                 })

        for doc in docs:
            annotations = []
            for annotation in DocumentAnnotation.all().filter("document =", doc.filename).run():
                anno_dict = {}
                anno_dict['annotator'] = annotation.user_id
                anno_dict['arg_units'] = annotation.arg_units
                anno_dict['relations'] = annotation.relations
                anno_dict['concepts'] = annotation.concepts
                anno_dict['approved'] = str(annotation.approved)
                anno_dict['notes'] = annotation.notes
                annotations.append(anno_dict)

            jsonResponse.append({'file': doc.filename,
                                 'text': doc.text,
                                 'url': doc.url,
                                 'user_annotations': annotations,
                                 'num_tokens': doc.num_tokens,
                                 'num_sentences': doc.num_sentences
                                 })

        dump_filename = "dump_" + time.strftime("%Y-%m-%d_%H:%M:%S") + ".json"
        self.response.headers['Content-Type'] = 'application/json'
        self.response.headers['Content-Disposition'] = "attachment; filename=%s" % dump_filename
        self.response.write(
            json.dumps(jsonResponse, indent=2, sort_keys=False, separators=(',', ':')))

    def load_documents(self, force_update=False):
        response_text = ""

        data_folder = 'data/'
        metadata_filename = data_folder + 'metadata.properties'

        if os.path.exists(metadata_filename):
            db.delete(CorpusMetadata.all())
            metadata_map = {}
            metadata_file = open(metadata_filename, 'r')
            for line in metadata_file.readlines():
                if not line.startswith("#"):
                    parts = line.split("=", 1)
                    if len(parts) is 2:
                        metadata_map[parts[0]] = parts[1].strip()
            metadata = extract_metadata(metadata_map)
            metadata.put()

        files = glob.glob(data_folder + '*.json')
        doc_ids = []
        new_documents = 0
        skipped_documents = 0
        updated_documents = 0
        dropped_documents = 0

        if force_update:
            response_text += "Update forced!\n"

        for f in sorted(files):
            basename = os.path.basename(f)
            jdata = json.load(open(f, 'r'))

            documents_with_same_url = Document.all().filter("url =", jdata['url'])
            is_document_in_datastore = 0 != documents_with_same_url.count()

            jdata['file'] = basename
            doc_ids.append(basename)

            if is_document_in_datastore:

                existing_doc = documents_with_same_url.get()

                if force_update:
                    initialize_document(existing_doc, jdata)
                    existing_doc.put()

                    response_text += 'UPDATED: ' + str(basename) + " " + str(jdata['url'])
                    updated_documents += 1

                else:

                    response_text += 'SKIPPED: ' + str(basename) + " " + str(jdata['url'])
                    skipped_documents += 1
            else:
                doc = Document()
                initialize_document(doc, jdata)

                doc.put()

                response_text += '    NEW: ' + str(basename) + " " + str(jdata['url'])
                new_documents += 1

            response_text += '\n'

        response_text += "----\n"

        if force_update:
            for document in Document.all():
                if document.filename not in doc_ids:
                    dropped_documents += 1
                    db.delete(document);
                    response_text += "DROPPED: " + document.filename + "\n"

        response_text += "=" * 100 + "\n"
        response_text += "Summary:\n"
        response_text += "\tNew:    " + str(new_documents) + "\n"
        response_text += "\tUpdated:" + str(updated_documents) + "\n"
        response_text += "\tSkipped:" + str(skipped_documents) + "\n"
        response_text += "\tDropped:" + str(dropped_documents) + "\n"

        return response_text

    def setApproval(self, annotator, document, isApproved):
        annotation = DocumentAnnotation.all().filter("user_id =", annotator).filter("document =",
                                                                                    document).get();
        if annotation:
            annotation.approved = isApproved
            annotation.put()

    def drop_all_data(self):
        db.delete(DocumentAnnotation.all().fetch(10000))
        db.delete(ArgumentationUnit.all().fetch(10000))
        db.delete(Document.all().fetch(10000))
        db.delete(UserData.all().fetch(10000))
        db.delete(CorpusMetadata.all().fetch(10000))

    def drop_all_annotations(self):
        db.delete(DocumentAnnotation.all().fetch(10000))
        db.delete(ArgumentationUnit.all().fetch(10000))
        db.delete(UserData.all().fetch(10000))

    def post(self):

        if not self.validate_user():
            return
        if self.request.path.endswith('importannotations'):
            dump_content = self.request.get("dumpfile")
            if dump_content:

                self.drop_all_annotations()
                verbatim_message = ""

                json_data = json.loads(dump_content)
                for doc_data in json_data:
                    if "corpus_metadata" in doc_data:
                        continue

                    filename = doc_data["file"]

                    doc = Document.all().filter("filename =", filename).get()
                    if doc:

                        arg_units = []
                        relations = []
                        concepts = []

                        for annotation_data in doc_data["user_annotations"]:
                            anno = self.annotation_from_json(annotation_data, doc)
                            arg_units.extend(anno.arg_units)
                            relations.extend(anno.relations)
                            concepts.extend(anno.concepts)
                            anno.put()

                        verbatim_message += "IMPORTED %25s: %4d arg. units\n" % (
                        filename, len(arg_units))
                    else:
                        verbatim_message += "SKIPPED  %25s: Document not in collection.\n" % (
                        filename)

                message = "Annotations imported."
            else:
                verbatim_message = ""
                message = "No file to import!"

            self.redirect('%s?%s' % (self.base_path(), urllib.urlencode(
                {"message": message, "verbatim_message": verbatim_message})))
        if self.request.path.endswith('importdump'):
            dump_content = self.request.get("dumpfile")
            if dump_content:

                self.drop_all_data()

                json_data = json.loads(dump_content)
                docs = []
                annos = []
                for doc_data in json_data:
                    if "corpus_metadata" in doc_data:
                        metadata = extract_metadata(doc_data)
                        metadata.put()
                    else:
                        doc = Document()
                        initialize_document(doc, doc_data)
                        docs.append(doc)

                        for annotation_data in doc_data["user_annotations"]:
                            anno = self.annotation_from_json(annotation_data, doc)
                            annos.append(anno)

                db.put(docs);
                db.put(annos);

                message = "Corpus dump imported."
            else:
                message = "No file to import!"

            self.redirect('%s?%s' % (self.base_path(), urllib.urlencode({"message": message})))

    def annotation_from_json(self, annotation_data, doc):
        anno = DocumentAnnotation()
        anno.approved = self.bool_from_string(
            annotation_data["approved"]) if "approved" in annotation_data else False
        anno.user_id = annotation_data["annotator"]

        anno.notes = annotation_data["notes"] if "notes" in annotation_data else ""

        if "arg_units" in annotation_data:
            anno.arg_units = [Text(p) for p in annotation_data["arg_units"]]
        else:
            anno.arg_units = [Text(p) for p in annotation_data["propositions"]]

        if "relations" in annotation_data:
            anno.relations = [Text(p) for p in annotation_data["relations"]]

        if "concepts" in annotation_data:
            anno.concepts = [Text(p) for p in annotation_data["concepts"]]

        self.convert_from_legacy_arg_units(anno)
        anno.document = doc.filename
        return anno

    def convert_from_legacy_arg_units(self, anno):
        """
        In former versions of the tools, pre-claim and post-claim premises were not distinguished.
        This function converts the arg_units of the annotation to pre-claim arg_units.
        """
        new_arg_units = []
        for arg_unit in anno.arg_units:
            new_arg_unit = str(arg_unit)

            # As of Nov 18, 2013, we distinguish pre- and post-claim premises
            new_arg_unit = new_arg_unit.replace("\"support\"", "\"support-pre\"")
            new_arg_unit = new_arg_unit.replace("\"rebutter\"", "\"rebutter-pre\"")
            # As of Nov 22, 2013, rebutters are called attack now
            new_arg_unit = new_arg_unit.replace("rebutter", "attack")

            # As of Nov 20, 2013, arg_units have a confidence level
            entries = new_arg_unit.split(",")
            if entries[1] not in ["\"high\"", "\"medium\"", "\"low\""]:
                entries.insert(1, DEFAULT_CONFIDENCE);

            new_arg_unit = ",".join(entries)

            new_arg_units.append(Text(new_arg_unit))
        anno.arg_units = new_arg_units

    def bool_from_string(self, s):
        if s == "True":
            return True
        else:
            return False


app = webapp2.WSGIApplication([('/argunit/managedata.*', ManageDataHandler)],
                              debug=True)
