# -------------------------------------------------------------------------------
#  Name:        module1
#  Purpose:
# 
#  Author:      Stefan
# 
#  Created:     03.08.2017
#  Copyright:   (c) Stefan 2017
#  Licence:     <your licence>
# -------------------------------------------------------------------------------

from urllib.parse import urljoin
from bs4 import BeautifulSoup
from scrape_interface import ScrapeProcessor
import re

# number of entity for which we download
_ENTITY = 4316422


def extractdata(sp):
    print("Start processing entity " + str(_ENTITY))
    _process_main(sp, "http://ps4.ro/consiliul-local/sedintele-consiliului-local/procese-verbale/", True)
    _process_main(sp, "http://ps4.ro/consiliul-local/hotarari/", False)
    print("End processing entity " + str(_ENTITY))


# main processing - take all years and process until no more pages
def _process_main(sp, configaddress, ispvpage):
    pageno = 1
    while _process_page(sp, configaddress, pageno, ispvpage):
        pageno = pageno + 1


# process a page from the site
# return True if there is some more data
def _process_page(sp, configaddress, pageno, ispvpage):
    html = ScrapeProcessor.download_page(configaddress + "?page_n=" + str(pageno))
    soup = BeautifulSoup(html, 'html.parser')
    tables = soup.find_all("table", {"class": "table table-hover course-list-table tablesorter"})
    if len(tables) == 0:
        return False
    for trow in tables[0].find_all("tr"):
        cells = trow.find_all("td")
        if len(cells) < 3:
            continue
        title = cells[0].string.strip()
        number = 0
        if ispvpage:
            datetext = ScrapeProcessor.finddate(cells[1].string)
            alink = cells[2].a
        else:
            number = cells[1].string.strip()
            datetext = ScrapeProcessor.finddate(cells[2].string)
            alink = cells[3].a
        if datetext == "":
            print("Error|no valid date " + cells[2].string)
            continue
        if alink is None:
            print("ERROR|no link|" + title + " " + datetext)
            continue
        href = alink["href"]
        if not href.startswith("http://"):
            href = urljoin("http://ps4.ro", href)
        if ispvpage:
            code, result = _process_pv(sp, href, datetext, title)
        else:
            code, result = _process_doc(sp, number, datetext[:4], title, href, datetext)
        if code == "ERROR":
            print("ERROR|" + result + "|" +  title + " " + datetext)
        if datetext[:4] < str(ScrapeProcessor.currentyear()) and \
            (sp.get_processmode() in (ScrapeProcessor.ProcessMode.DELTA, ScrapeProcessor.ProcessMode.DELTA_DOWNLOAD)):
            return False
    return True


# process the info regarding a document (decision)
# decision info should come in docInfo with the following tags:
# date, link, number, year, title
def _process_doc(sp, number, year, title, link, date):
    decisiontype = "HOTA"
    # for 2012 and 2016 the town hall started renaming the decisions after elections
    if date.startswith("2012") or date.startswith("2016"):
        if date[5:7] >= "06":
            decisiontype = "HOTB"

    # add the decision to server
    code, result = sp.post_decision(decisiontype, number, year, _ENTITY, date, title)
    if code == "ERROR":
        return code, result
    decisionid = result

    # download page
    code, result = sp.download_file(link)
    if code == "ERROR":
        sp.post_document("MAIN", decisionid, 0, "ERROR_DOWNLOAD", "", link)
        return code, result
    fname = result
    if code == "ARCHIVE":
        code, result = sp.unpack_zip(fname)
        if code == "ERROR":
            return code, result
        filelist = result
        response = "CREATED"
        annex = 0
        for fileInfo in filelist:

            doctype = "MAIN"
            if re.search("anexa", fileInfo[0], re.IGNORECASE) or re.search("-a[1-9]", fileInfo[0], re.IGNORECASE):
                annex = annex + 1
                doctype = "ANEX"

            fname = fileInfo[1]
            code, result, filetype = sp.ocr_document(fname)
            if code == "ERROR":
                sp.post_document(doctype, decisionid, annex, "ERROR_OCR", "", link)
                print("ERROR|" + result)
                response = "ERROR"
                continue
            ocrfname = result
            outstr, cssstr = ScrapeProcessor.preparehtml(ocrfname, filetype)

            code, result = sp.post_document(doctype, decisionid, annex, outstr, cssstr, link)
            if code == "ERROR":
                print("ERROR|" + result)
                response = "ERROR"
        return response, decisionid
    else:
        code, result, filetype = sp.ocr_document(fname)
        if code == "ERROR":
            sp.post_document("MAIN", decisionid, 0, "ERROR_OCR", "", link)
            return code, result
        ocrfname = result
        outstr, cssstr = ScrapeProcessor.preparehtml(ocrfname, filetype)
        return sp.post_document("MAIN", decisionid, 0, outstr, cssstr, link)


# process the info regarding a document (decision)
# decision info should come in docInfo with the following tags:
# date, link, number, year, title
def _process_pv(sp, link, date, title):
    number = ScrapeProcessor.dayinyear(date)
    code, result = sp.post_decision("PRVB", number, date[:4], _ENTITY, date, title)
    if code == "ERROR":
        return code, result
    decisionid = result

    # download page
    code, result = sp.download_file(link)
    if code == "ERROR":
        sp.post_document("MAIN", decisionid, 0, "ERROR_DOWNLOAD", "", link)
        return code, result
    fname = result
    code, result, filetype = sp.ocr_document(fname)
    if code == "ERROR":
        sp.post_document("MAIN", decisionid, 0, "ERROR_OCR", "", link)
        return code, result
    ocrfname = result
    outstr, cssstr = ScrapeProcessor.preparehtml(ocrfname, filetype)
    return sp.post_document("MAIN", decisionid, 0, outstr, cssstr, link)


if __name__ == '__main__':
    localsp = ScrapeProcessor("http://192.168.56.10", "stefan_cioc", "parola1234")
    localsp.set_folders("X:/hot/S4I", "X:/hot/S4O")
    localsp.set_processmode(ScrapeProcessor.ProcessMode.SIMULATE)
    extractdata(localsp)
