# This file replaces `process_questions_content_data.py`

# Copyright 2017 James Sharkey
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# CREATE TABLE public.content_data (
#     question_id text NOT NULL, page_id text NOT NULL,
#     title text, type text, published boolean,
#     subject text, field text, topic text, level integer,
#     hidden boolean, superseded boolean, problem_solving boolean,
#     book boolean, book_id text,
#     fasttrack boolean, fasttrack_id text, extraordinary boolean,
#     concept_page boolean, natsci_ia boolean,
#     workbook boolean, workshop boolean,
#     video_id text, hints integer,
#     hint_types jsonb, hint_apps jsonb,
#     related_questions jsonb, related_concepts jsonb,
#     detailed_concept_sections jsonb,
#     attribution text, json_path text,
# CONSTRAINT primarykey_content_data PRIMARY KEY (question_id));
#
# COPY content_data FROM 'content_data_2017.csv' DELIMITER ',' CSV HEADER;

import json
import os
import re
import sys
import csv
import datetime
from collections import OrderedDict

# Nasty hack to force utf-8 encoding by default:
reload(sys)
sys.setdefaultencoding('utf8')

# Useful settings:
_TOP_LEVEL_DIRECTORY = "C:/Path/To/Repository/Parent"
_CONTENT_DIRECTORY = "./rutherford-content/content"
ignore_biology = True


video_regex = re.compile(r'([\&\?]v=|/embed/)([A-Za-z0-9\-_]+)')
concepts_acc_regex = re.compile(r'(?<=\/concepts\/)([A-z0-9_]+#[A-z0-9_]+)(?=[\)\}])')

SUBJECTS = set(["physics", "maths", "chemistry"])
FIELDS = set(["mechanics", "waves", "fields", "circuits", "geometry", "calculus",
              "algebra", "functions", "chemphysics"])
TOPICS = set(["dynamics", "kinematics", "statics", "circular_motion", "shm",
              "angular_motion", "geom_vectors", "trigonometry", "shapes", "symmetry",
              "integration", "differentiation", "differential_eq", "simultaneous",
              "quadratics", "manipulation", "series", "special", "trigonometric",
              "means", "distributions", "electric", "magnetic", "gravitational",
              "combined", "superposition", "wave_motion", "optics", "resistors",
              "capacitors", "general_circuits", "components", "thermodynamics",
              "graph_sketching", "general_functions"])
Q_TYPES = {"isaacQuestion": "quick", "isaacNumericQuestion": "num", "isaacMultiChoiceQuestion": "multi", "isaacSymbolicQuestion": "sym",
           "isaacSymbolicChemistryQuestion": "chem"}
TO_HUMAN = {"fasttrack": "FastTrack Maths", "shm": "SHM", "chemphysics": "Physical Chemistry",
            "differential_eq": "Differential Equations", "geom_vectors": "Vectors", "electric": "Electric Fields",
            "magnetic": "Magnetic Fields", "combined": "Combined Fields",
            "physics_skills_14": "Physics A-Level", "phys_book_gcse": "Physics GCSE", "chemistry_16": "Chemistry A-Level"}

Questions = OrderedDict()
Question_Pages = OrderedDict()

QUESTION_MAP = dict()
CONCEPT_MAP = dict()
CONCEPT_SECTION_MAP = dict()


class ContentDataRow(object):

    def __init__(self, question_id=None, title=None, q_type=None, published=False,
                 subject=None, field=None, topic=None, level=None, hidden=False,
                 problem_solving=False, book=False, book_id=None, fasttrack_id=None,
                 extraordinary=False, concept_page=False, natsci_ia=False, workbook=False, workshop=False,
                 video_id=None, hints=0, hint_types=None, hint_apps=None, related_content=None,
                 superseded_by=None, detailed_concept_sections=None, attribution=None,
                 json_path=None):
        self.question_id = question_id
        self.page_id = question_id.split("|")[0]
        self.title = title
        self.q_type = q_type
        self.published = published
        self.subject = subject
        self.field = field
        self.topic = topic
        self.level = level
        self.hidden = hidden
        self.superseded_by = superseded_by
        self.problem_solving = problem_solving
        self.book = book
        self.book_id = book_id
        self.fasttrack = fasttrack_id is not None
        self.fasttrack_id = fasttrack_id
        self.extraordinary = extraordinary
        self.concept_page = concept_page
        self.natsci_ia = natsci_ia
        self.workbook = workbook
        self.workshop = workshop
        self.video_id = video_id
        self.hints = hints
        self.hint_types = hint_types
        self.hint_apps = hint_apps
        self.related_content = []
        self.detailed_concept_sections = detailed_concept_sections
        self.attribution = attribution
        self.json_path = json_path
        #
        self.related_questions = []
        self.related_concepts = []
        if related_content is not None:
            self.set_related_content(related_content)

    def __repr__(self):
        return "<isaacQuestion: %s>" % self.question_id

    def __str__(self):
        return repr(self)

    def set_related_content(self, related_content):
        self.related_content = sorted(related_content)
        self.related_concepts = sorted(set(CONCEPT_MAP.keys()).intersection(related_content))
        self.related_questions = sorted(set(QUESTION_MAP.keys()).intersection(related_content))

    def get_database_row(self):
        return [self.question_id, self.page_id, self.title, self.q_type, self.published,
                self.subject, self.field, self.topic, self.level, self.hidden, self.superseded_by is not None,
                self.problem_solving, self.book, self.book_id, self.fasttrack, self.fasttrack_id, self.extraordinary,
                self.concept_page, self.natsci_ia, self.workbook, self.workshop, self.video_id,
                self.hints, self.jsonify(self.hint_types), self.jsonify(self.hint_apps),
                self.jsonify(self.related_questions), self.jsonify(self.related_concepts),
                self.jsonify(self.detailed_concept_sections), self.attribution, self.json_path]

    @staticmethod
    def get_database_title_row():
        return ['question_id', 'page_id', 'title', 'q_type', 'published',
                'subject', 'field', 'topic', 'level', 'hidden', 'superseded',
                'problem_solving', 'book', 'book_id', 'fasttrack', 'fasttrack_id', 'extraordinary',
                'concept_page', 'natsci_ia', 'workbook', 'workshop', 'video_id', 'hints',
                'hint_types', 'hint_apps', 'related_questions', 'related_concepts',
                'detailed_concept_sections', 'attribution', 'json_path']

    @staticmethod
    def jsonify(thing):
        return json.dumps(thing).replace("null", "").replace("[]", "")


class QuestionPageEntry(object):

    def __init__(self, page_id, content_data_rows):
        self.page_id = page_id
        self.title = QUESTION_MAP[page_id][0] if page_id in QUESTION_MAP else ""
        self.level = content_data_rows[0].level
        self.subject = self.human_readable(content_data_rows[0].subject)
        self.field = self.human_readable(content_data_rows[0].field)
        self.topic = self.human_readable(content_data_rows[0].topic)
        self.superseded = content_data_rows[0].superseded_by is not None
        ##
        self.book_id = content_data_rows[0].book_id
        self.fasttrack = content_data_rows[0].fasttrack
        ##
        self.public = all([(r.published and not r.hidden and (r.book or r.problem_solving or r.fasttrack)) for r in content_data_rows])
        self.any_hints = any([r.hints > 0 for r in content_data_rows])
        self.diagrams_required = any(["DIAGRAM" in r.hint_types for r in content_data_rows if r.hint_types is not None])
        self.videos = any(["VIDEO" in r.hint_types for r in content_data_rows if r.hint_types is not None])
        self.q_types = [r.q_type for r in content_data_rows]
        self.related_questions = None
        self.related_concepts = None
        ##
        self.related_questions = sorted([QUESTION_MAP[e][0] for e in content_data_rows[0].related_questions if QUESTION_MAP[e][1]])
        self.related_concepts = sorted([CONCEPT_MAP[e][0] for e in content_data_rows[0].related_concepts if CONCEPT_MAP[e][1]])
        concept_sections = set([c for r in content_data_rows if r.detailed_concept_sections is not None for c in r.detailed_concept_sections])
        self.detailed_concepts = self.title_detailed_concepts(concept_sections)

    def get_csv_row(self):
        entry = []
        entry.append(self.title.strip())
        entry.append("/questions/" + self.page_id)
        entry.append(self.human_readable(self.book_id))
        entry.append(self.human_readable(self.fasttrack))
        entry.append(self.subject)
        entry.append(self.human_readable(self.level))
        entry.append(self.field)
        entry.append(self.topic)
        entry.append(self.human_readable(len(self.q_types)))
        entry.append(self.human_readable("quick" in self.q_types))
        entry.append(self.human_readable("multi" in self.q_types))
        entry.append(self.human_readable("num" in self.q_types))
        entry.append(self.human_readable("sym" in self.q_types or "chem" in self.q_types))
        entry.append(self.human_readable(self.any_hints))
        entry.append(self.human_readable(self.videos))
        entry.append(self.human_readable(self.diagrams_required))
        entry.append(self.human_readable(self.related_questions))
        entry.append(self.human_readable(self.related_concepts))
        entry.append(self.human_readable(self.detailed_concepts))
        return entry

    @staticmethod
    def get_csv_header_rows(date_string):
        date_line = ["File created on " + date_string + "."]
        disclaimer = ["This information is provded as is. Please let us know of any mistakes or inaccuracies."]
        columns = ["Question Name:", "URL:", "Book Name?", "Maths FastTrack?", "Subject:",
                   "Level:", "Field:", "Topic:", "Number of Parts:",
                   "Contains Show/Hide Q?", "Contains Multi Choice Q?", "Contains Numeric Q?", "Contains Symbolic Q?",
                   "Has Hints?", "Has Video?", "Has Diagram?", "Related Questions:", "Related Concepts:", "Detailed Concept Sections:"]
        return [date_line, disclaimer, [], columns]

    @staticmethod
    def title_detailed_concepts(concept_sections):
        concepts = OrderedDict()
        for cs in concept_sections:
            c, s = cs.split("#")
            if (c not in CONCEPT_MAP) or (not CONCEPT_MAP[c][1]):
                # This isn't public or it's not valid concept page.
                continue
            if (c not in CONCEPT_SECTION_MAP) or (s not in CONCEPT_SECTION_MAP[c]):
                # This isn't a valid accordion section.
                continue
            if c not in concepts:
                concepts[c] = []
            concepts[c].append(CONCEPT_SECTION_MAP[c][s])
        concepts = {c: ", ".join(sorted(v)) for c, v in concepts.items()}
        concept_titles = ["%s (%s)" % (CONCEPT_MAP[page][0], sections) for page, sections in concepts.items()]
        return sorted(concept_titles)

    @staticmethod
    def human_readable(machine_readable):
        # If it's a string, change to title case and replace any weird names:
        if type(machine_readable) is str or type(machine_readable) is unicode:
            if machine_readable in TO_HUMAN:
                human_read = TO_HUMAN[machine_readable]
            else:
                human_read = machine_readable.replace("_", " ").title()
        # If it's a bool, decide how to represent it:
        elif type(machine_readable) is bool:
            if machine_readable:
                human_read = "Y"
            else:
                human_read = "N"
        # If it's an int, represent 0 differently:
        elif type(machine_readable) is int:
            if machine_readable == 0:
                human_read = ""  # "N/A"
            else:
                human_read = str(machine_readable)
        # If it's a list, join it:
        elif type(machine_readable) is list:
            human_read = ", ".join(machine_readable)
        # If it's None, make it blank:
        elif machine_readable is None:
            human_read = ""
        # Otherwise, just make it a string:
        else:
            human_read = str(machine_readable)
        return human_read


def get_date_string():
    date_run = datetime.datetime.today()
    date_string = date_run.strftime("%A, %d %B %Y").replace(" 0", " ")  # Don't zero-pad days!
    # Add in a "st, nd, rd, th" as required:
    date_string = date_string.replace("11 ", "11th ").replace("12 ", "12th ").replace("13 ", "13th ")  # Deal with awkward teens
    date_string = date_string.replace("1 ", "1st ").replace("2 ", "2nd ").replace("3 ", "3rd ")  # Deal with #1st #2nd and #3rd
    date_string = re.sub("([0-9]) ", "\g<1>th ", date_string)  # And all the rest get "th"
    return date_string


def find_app_id(h_obj):
    if ("type" in h_obj) and (h_obj["type"] == "anvilApp"):
        return h_obj["appId"]

    for c in h_obj.get("children", []):
        a_id = find_app_id(c)
        if a_id is not None:
            # This assumes there's only ever one app!
            return str(a_id)


def find_video(h_obj):
    if ("type" in h_obj) and (h_obj["type"] == "video"):
        m = video_regex.search(h_obj["src"])
        if m is not None:
            return str(m.group(2))

    for c in h_obj.get("children", []):
        v = find_video(c)
        if v is not None:
            # This assumes there's only ever one video!
            return v


def flatten_types(obj):
    types = set()
    if "type" in obj:
        types.add(obj["type"])
    for c in obj.get("children", []):
        types = types.union(flatten_types(c))
    return types


def flatten_values(obj):
    value = ""
    if "value" in obj:
        value += (" " + obj["value"])
    for c in obj.get("children", []):
        value += (" " + flatten_values(c))
    return value


def process_hints(hints):
    apps = dict()
    hint_types = []
    detailed_concepts = None
    video = None
    num_hints = len(hints)
    for index, h in enumerate(hints):
        subtypes = flatten_types(h)
        text = flatten_values(h)
        if "anvilApp" in subtypes:
            hint_types.append("ANVIL_APP")
            apps[index] = find_app_id(h)
        elif "video" in subtypes:
            hint_types.append("VIDEO")
            video = video or find_video(h)
        elif "figure" in subtypes:
            hint_types.append("DIAGRAM")
        elif "**Concepts**" in text:
            hint_types.append("CONCEPTS")
            detailed_concepts = concepts_acc_regex.findall(text)
        elif ("**Goal**" in text or "**Information given**" in text or "**Information Given**" in text or
              "**Glossary**" in text or "**Useful information" in text or "**Information assumed**" in text):
            hint_types.append("INFO")
        elif ("**Useful equations**" in text or "** Useful equations **" in text):
            hint_types.append("EQUATIONS")
        elif ("**Useful tip" in text or "**Useful questions**" in text or "**Useful mathematics**" in text):
            hint_types.append("TIP")
        elif (re.search(r"<a href=[\"']/pages/gcse_ch[0-9]+_[0-9]+_text", text) is not None):
            hint_types.append("GCSE_SECTION_TEXT")
        else:
            hint_types.append("UNKNOWN")
    if len(apps.keys()) == 0:
        apps = None
    if len(hint_types) == 0:
        hint_types = None
    return num_hints, video, hint_types, apps, detailed_concepts


def process_question(json_path, obj, _id, published, tags, level, related_content, attribution, superseded_by):
    if ("type" not in obj) or ("id" not in obj):
        print "JSON Parse Error in Question: %s !!!" % json_path
        return
    if _id in Questions:
        print "Duplicate Question ID in %s !!!" % json_path
        return

    # Build the ContentDataRow object using bare minimum and update later:
    r = ContentDataRow(question_id=_id, q_type=Q_TYPES[obj["type"]], json_path=json_path)
    # Keep track of the ContentDataRows
    if r.page_id not in Question_Pages:
        Question_Pages[r.page_id] = []
    Questions[r.question_id] = r
    Question_Pages[r.page_id].append(r)

    if "title" in obj:
        r.title = obj["title"]
    r.published = published
    r.level = level
    r.attribution = attribution
    r.superseded_by = superseded_by
    if tags is not None:
        r.subject = ", ".join(sorted(SUBJECTS.intersection(tags)))
        if r.page_id in QUESTION_MAP:  # Concept pages don't really have fields and topics!
            r.field = ", ".join(sorted(FIELDS.intersection(tags)))
            r.topic = ", ".join(sorted(TOPICS.intersection(tags)))
        if "nofilter" in tags:
            r.hidden = True
    if "/books/" in json_path and json_path.endswith(".json"):
        r.book = True
        r.book_id = json_path.split("/books/")[1].split("/")[0]
    elif "/fasttrack/" in json_path:
        r.fasttrack = True
        if "c1_c2_2017" in json_path:
            r.fasttrack_id = "ft_core_2017"
        elif "c1_c2" in json_path:
            r.fasttrack_id = "fasttrack_core_1_14"
        elif "c3_c4" in json_path:
            r.fasttrack_id = "fasttrack_core_3_15"
        elif "/mechanics/m1/" in json_path:
            r.fasttrack_id = "fasttrack_core_2_14"
        elif "/mechanics/m1/" in json_path:
            r.fasttrack_id = "ft_mechanics2"
    elif "/featured_problems/extraordinary_problems/" in json_path:
        r.extraordinary = True
    elif (tags is not None and "nst_workbook" in tags):
        r.workbook = True
    elif "/workshop/" in json_path:
        r.workshop = True
    elif ("/questions/physics/" in json_path or "/questions/maths/" in json_path):
        r.problem_solving = True
    elif "/concepts/" in json_path:
        r.concept_page = True
    elif "/NSTIA/" in json_path:
        r.natsci_ia = True
    if "hints" in obj:
        num_hints, video_id, hint_types, hint_apps, concepts = process_hints(obj["hints"])
        r.video_id = video_id
        r.hint_types = hint_types
        r.hint_apps = hint_apps
        r.hints = num_hints
        r.detailed_concept_sections = concepts
    if (related_content is not None and len(related_content) > 0):
        r.set_related_content(related_content)


def process_thing(json_path, obj, _id=None, published=False, tags=None, level=None, rel=None, attr=None, superseded_by=None):
    if ("type" not in obj):
        print "JSON Parse Error: %s !!!" % json_path
        return

    if "id" in obj:
        if _id is None:
            _id = obj["id"]
        else:
            _id += ("|%s" % obj["id"])

    if "published" in obj:
        published = obj["published"]

    if "tags" in obj:
        tags = obj["tags"]

    if "level" in obj:
        level = obj["level"]

    if "relatedContent" in obj:
        rel = map(str, obj["relatedContent"])

    if "attribution" in obj:
        attr = obj["attribution"]

    if "supersededBy" in obj:
        superseded_by = obj["supersededBy"]

    if obj["type"] in Q_TYPES.keys():
        # Assume questions cannot be nested
        return process_question(json_path, obj, _id, published, tags, level, rel, attr, superseded_by)

    if "children" in obj:
        for c in obj["children"]:
            process_thing(json_path, c, _id, published, tags, level, rel, attr, superseded_by)


def extract_accordion_titles(obj, accordion=False):
    id_title_map = dict()
    if "type" not in obj:
        return id_title_map
    if accordion:  # If we're the children of an accordion, our titles and IDs are sections!
        if "id" in obj:
            if "title" in obj:
                t = obj["title"].strip().replace("  ", " ")
                t = re.sub(r'\\frac{(.*?)}{(.*?)}', "\g<1> / \g<2>", t)
                t = re.sub(r'\\valuedef{(.*?)}{(.*?)}', "\g<1> = \g<2>", t)
                t = t.replace("\\vari{", "").replace("\\langle", "<").replace("\\rangle", ">")
                t = t.replace("$", "").replace("\\", "").replace("{", "").replace("}", "")
                id_title_map[obj["id"]] = t
            else:
                id_title_map[obj["id"]] = ""
            # Assume that accordions are not nested!
            return id_title_map
    if "children" in obj:
        # Whether or not we're the children of an accordion is important:
        accordion = obj["type"] == "content" and "layout" in obj and obj["layout"] == "accordion"
        for c in obj["children"]:
            id_title_map.update(extract_accordion_titles(c, accordion))
    return id_title_map


def extract_question_concept_title(obj, question_map, concept_map, concept_section_map):
    if "type" not in obj or "title" not in obj or "id" not in obj:
        return
    # Is it live?
    published = "published" in obj and obj["published"]
    # Add to the right map:
    if obj["type"] in ["isaacQuestionPage", "isaacFastTrackQuestionPage"]:
        public = published and ("tags" not in obj or "nofilter" not in obj["tags"])
        question_map[obj["id"]] = (obj["title"], public)
    elif obj["type"] == "isaacConceptPage":
        public = published
        concept_map[obj["id"]] = (obj["title"], public)
        accordion_sections = extract_accordion_titles(obj)
        if len(accordion_sections) > 0:
            concept_section_map[obj["id"]] = accordion_sections


###############################################################################
# The main loops:

# Find all relevant JSON files in the _CONTENT_DIRECTORY:
os.chdir(_TOP_LEVEL_DIRECTORY)
all_files = []
for root, directories, files in os.walk(_CONTENT_DIRECTORY):
    for fname in files:
        fpath = root + "\\" + fname
        if (not ("biology" in fpath.lower()) or not ignore_biology):
            if fpath.lower().endswith(".json"):
                all_files.append(fpath)

# Find titles of question and concept pages:
for fpath in all_files:
    with open(fpath) as file_handle:
        try:
            obj = json.load(file_handle)
        except UnicodeDecodeError as e:
            print fpath
            raise
        extract_question_concept_title(obj, QUESTION_MAP, CONCEPT_MAP, CONCEPT_SECTION_MAP)

# For each JSON file, parse and process it:
for fpath in all_files:
    with open(fpath) as file_handle:
        try:
            obj = json.load(file_handle)
        except UnicodeDecodeError as e:
            print fpath
            raise
        json_path = "/".join(fpath.split("\\")).replace("./rutherford-content/content", "")
        process_thing(json_path, obj)


# Write out the database question data CSV file:
outfile = open("content_data_2017.csv", 'wb')
writer = csv.writer(outfile)
writer.writerow(ContentDataRow.get_database_title_row())
for q in Questions.values():
    writer.writerow(q.get_database_row())
outfile.close()

# Write out the question page information CSV file:
outfile = open("isaac_physics_questions.csv", 'wb')
writer = csv.writer(outfile)
writer.writerows(QuestionPageEntry.get_csv_header_rows(get_date_string()))

question_ids_to_export = sorted(set(Question_Pages.keys()).intersection(QUESTION_MAP.keys()))
questions_to_export = [QuestionPageEntry(q, Question_Pages[q]) for q in question_ids_to_export]
#     (Do a horrible amount of sorting, to get things in a nice order!)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.title)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.level)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.topic)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.field)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.subject, reverse=True)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.book_id)
questions_to_export = sorted(questions_to_export, key=lambda QE: QE.fasttrack)
for question_page in questions_to_export:
    if not question_page.public:
        continue
    writer.writerow(question_page.get_csv_row())
outfile.close()
