import datetime
import os
import pymongo
import requests
from urllib import parse
from urllib.parse import quote_plus
import json
from retry import retry

from bs4 import BeautifulSoup
import math

from project.server.main.logger import get_logger
from project.server.main.utils_swift import upload_object
from project.server.main.parse import parse_theses, get_idref_from_OS
from project.server.main.referentiel import harvest_and_save_idref

logger = get_logger(__name__)

def get_num_these(soup):
    num_theses = []
    for d in soup.find_all('doc'):
        num_theses.append(d.find('str', {'name': 'num'}).text)
    return num_theses

@retry(delay=60, tries=5)
def get_num_these_between_dates(start_date, end_date):
    start_date_str = start_date.strftime("%d/%m/%Y")
    end_date_str = end_date.strftime("%d/%m/%Y")

    start_date_str_iso = start_date.strftime("%Y%m%d")
    end_date_str_iso = end_date.strftime("%Y%m%d")

    start = 0


    url = "http://theses.fr/?q=&zone1=titreRAs&val1=&op1=AND&zone2=auteurs&val2=&op2=AND&zone3=etabSoutenances&val3=&op3=AND&zone4=dateSoutenance&val4a={}&val4b={}&start={}&format=xml"
    logger.debug(url.format(start_date_str, end_date_str, start))

    r = requests.get(url.format(start_date_str, end_date_str, start))

    soup = BeautifulSoup(r.text, 'lxml')

    nb_res = soup.find('result', {'name': 'response'}).attrs['numfound']
    logger.debug("{} resultats entre {} et {}".format(nb_res, start_date_str_iso, end_date_str_iso ))
    num_theses = get_num_these(soup)

    nb_pages_remaining = math.ceil(int(nb_res)/1000)
    for p in range(1, nb_pages_remaining):
        logger.debug("page {} for entre {} et {}".format(p, start_date_str_iso, end_date_str_iso))
        r = requests.get(url.format(start_date_str, end_date_str, p * 1000))
        soup = BeautifulSoup(r.text, 'lxml')
        num_theses += get_num_these(soup)
        
    return num_theses


def save_data(data, collection_name, year_start, year_end, chunk_index, referentiel):
    logger.debug(f'save_data theses {collection_name} {chunk_index}')
    year_start_end = 'all_years'
    if year_start and year_end:
        year_start_end = f'{year_start}_{year_end}'
    # 1. save raw data to OS
    current_file = f'theses_{year_start_end}_{chunk_index}.json'
    json.dump(data, open(current_file, 'w'))
    os.system(f'gzip {current_file}')
    upload_object('theses', f'{current_file}.gz', f'{collection_name}/raw/{current_file}.gz')
    os.system(f'rm -rf {current_file}.gz')

    # 2.transform data and save in mongo
    current_file_parsed = f'theses_parsed_{year_start_end}_{chunk_index}.json'
    data_parsed = [parse_theses(e, referentiel, collection_name) for e in data]
    json.dump(data_parsed, open(current_file_parsed, 'w'))
    # insert_data(collection_name, current_file_parsed)
    os.system(f'gzip {current_file_parsed}')
    upload_object('theses', f'{current_file_parsed}.gz', f'{collection_name}/parsed/{current_file_parsed}.gz')
    os.system(f'rm -rf {current_file_parsed}.gz')

def harvest_and_insert(collection_name):
    # 1. save aurehal structures
    harvest_and_save_idref(collection_name)
    referentiel = get_idref_from_OS(collection_name)

    # 2. drop mongo 
    #logger.debug(f'dropping {collection_name} collection before insertion')
    #myclient = pymongo.MongoClient('mongodb://mongo:27017/')
    #myclient['theses'][collection_name].drop()

    # 3. save publications
    year_start = None
    year_end = None
    if year_start is None:
        year_start = 1990
    if year_end is None:
        year_end = datetime.date.today().year
    harvest_and_insert_one_year(collection_name, year_start, year_end, referentiel)

@retry(delay=60, tries=5)
def download_these_notice(these_id):

    res = {'id': these_id}
    r_tefudoc = requests.get("http://www.theses.fr/{}.tefudoc".format(these_id))
    r_xml = requests.get("http://www.theses.fr/{}.xml".format(these_id))

    if r_tefudoc.text[0:5] == "<?xml":
        res['tefudoc'] = r_tefudoc.text

    if r_xml.text[0:5] == "<?xml":
        res['xml'] = r_xml.text

    return res

def harvest_and_insert_one_year(collection_name, year_start, year_end, referentiel):
    
    year_start_end = 'all_years'
    if year_start and year_end:
        year_start_end = f'{year_start}_{year_end}'

    start_date = datetime.datetime(year_start,1,1)
    end_date = datetime.datetime(year_end + 1,1,1) + datetime.timedelta(days = -1)

    all_num_theses = get_num_these_between_dates(start_date, end_date)

    # todo save by chunk
    chunk_index = 0
    data = []
    MAX_DATA_SIZE = 25000
    nb_theses = len(all_num_theses)
    logger.debug(f'{nb_theses} theses to download and parse')
    for ix, nnt in enumerate(all_num_theses):
        if ix % 100 == 0:
            logger.debug(f'theses {year_start_end} {ix}')
        res = download_these_notice(nnt)
        data.append(res)
        
        if (len(data) > MAX_DATA_SIZE) or (ix == nb_theses - 1):
            if data:
                save_data(data, collection_name, year_start, year_end, chunk_index, referentiel)
                data = []
                chunk_index += 1


def insert_data(collection_name, output_file):
    myclient = pymongo.MongoClient('mongodb://mongo:27017/')
    mydb = myclient['theses']
    
    ## mongo start
    start = datetime.datetime.now()
    mongoimport = f"mongoimport --numInsertionWorkers 2 --uri mongodb://mongo:27017/theses --file {output_file}" \
                  f" --collection {collection_name} --jsonArray"
    logger.debug(f'Mongoimport {output_file} start at {start}')
    logger.debug(f'{mongoimport}')
    os.system(mongoimport)
    logger.debug(f'Checking indexes on collection {collection_name}')
    mycol = mydb[collection_name]
    #mycol.create_index('docid')
    end = datetime.datetime.now()
    delta = end - start
    logger.debug(f'Mongoimport done in {delta}')
    ## mongo done

[((18, 9, 18, 29), 'project.server.main.logger.get_logger', 'get_logger', ({(18, 20, 18, 28): '__name__'}, {}), '(__name__)', False, 'from project.server.main.logger import get_logger\n'), ((26, 1, 26, 25), 'retry.retry', 'retry', (), '', False, 'from retry import retry\n'), ((98, 1, 98, 25), 'retry.retry', 'retry', (), '', False, 'from retry import retry\n'), ((42, 11, 42, 40), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(42, 25, 42, 31): 'r.text', (42, 33, 42, 39): '"""lxml"""'}, {}), "(r.text, 'lxml')", False, 'from bs4 import BeautifulSoup\n'), ((66, 4, 66, 37), 'os.system', 'os.system', ({(66, 14, 66, 36): 'f"""gzip {current_file}"""'}, {}), "(f'gzip {current_file}')", False, 'import os\n'), ((67, 4, 67, 93), 'project.server.main.utils_swift.upload_object', 'upload_object', ({(67, 18, 67, 26): '"""theses"""', (67, 28, 67, 48): 'f"""{current_file}.gz"""', (67, 50, 67, 92): 'f"""{collection_name}/raw/{current_file}.gz"""'}, {}), "('theses', f'{current_file}.gz',\n    f'{collection_name}/raw/{current_file}.gz')", False, 'from project.server.main.utils_swift import upload_object\n'), ((68, 4, 68, 42), 'os.system', 'os.system', ({(68, 14, 68, 41): 'f"""rm -rf {current_file}.gz"""'}, {}), "(f'rm -rf {current_file}.gz')", False, 'import os\n'), ((75, 4, 75, 44), 'os.system', 'os.system', ({(75, 14, 75, 43): 'f"""gzip {current_file_parsed}"""'}, {}), "(f'gzip {current_file_parsed}')", False, 'import os\n'), ((76, 4, 76, 110), 'project.server.main.utils_swift.upload_object', 'upload_object', ({(76, 18, 76, 26): '"""theses"""', (76, 28, 76, 55): 'f"""{current_file_parsed}.gz"""', (76, 57, 76, 109): 'f"""{collection_name}/parsed/{current_file_parsed}.gz"""'}, {}), "('theses', f'{current_file_parsed}.gz',\n    f'{collection_name}/parsed/{current_file_parsed}.gz')", False, 'from project.server.main.utils_swift import upload_object\n'), ((77, 4, 77, 49), 'os.system', 'os.system', ({(77, 14, 77, 48): 'f"""rm -rf {current_file_parsed}.gz"""'}, {}), "(f'rm -rf {current_file_parsed}.gz')", False, 'import os\n'), ((81, 4, 81, 43), 'project.server.main.referentiel.harvest_and_save_idref', 'harvest_and_save_idref', ({(81, 27, 81, 42): 'collection_name'}, {}), '(collection_name)', False, 'from project.server.main.referentiel import harvest_and_save_idref\n'), ((82, 18, 82, 52), 'project.server.main.parse.get_idref_from_OS', 'get_idref_from_OS', ({(82, 36, 82, 51): 'collection_name'}, {}), '(collection_name)', False, 'from project.server.main.parse import parse_theses, get_idref_from_OS\n'), ((119, 17, 119, 50), 'datetime.datetime', 'datetime.datetime', ({(119, 35, 119, 45): 'year_start', (119, 46, 119, 47): '1', (119, 48, 119, 49): '1'}, {}), '(year_start, 1, 1)', False, 'import datetime\n'), ((144, 15, 144, 60), 'pymongo.MongoClient', 'pymongo.MongoClient', ({(144, 35, 144, 59): '"""mongodb://mongo:27017/"""'}, {}), "('mongodb://mongo:27017/')", False, 'import pymongo\n'), ((148, 12, 148, 35), 'datetime.datetime.now', 'datetime.datetime.now', ({}, {}), '()', False, 'import datetime\n'), ((153, 4, 153, 26), 'os.system', 'os.system', ({(153, 14, 153, 25): 'mongoimport'}, {}), '(mongoimport)', False, 'import os\n'), ((157, 10, 157, 33), 'datetime.datetime.now', 'datetime.datetime.now', ({}, {}), '()', False, 'import datetime\n'), ((52, 15, 52, 44), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(52, 29, 52, 35): 'r.text', (52, 37, 52, 43): '"""lxml"""'}, {}), "(r.text, 'lxml')", False, 'from bs4 import BeautifulSoup\n'), ((72, 19, 72, 64), 'project.server.main.parse.parse_theses', 'parse_theses', ({(72, 32, 72, 33): 'e', (72, 35, 72, 46): 'referentiel', (72, 48, 72, 63): 'collection_name'}, {}), '(e, referentiel, collection_name)', False, 'from project.server.main.parse import parse_theses, get_idref_from_OS\n'), ((120, 15, 120, 50), 'datetime.datetime', 'datetime.datetime', ({(120, 33, 120, 45): '(year_end + 1)', (120, 46, 120, 47): '(1)', (120, 48, 120, 49): '(1)'}, {}), '(year_end + 1, 1, 1)', False, 'import datetime\n'), ((120, 53, 120, 82), 'datetime.timedelta', 'datetime.timedelta', (), '', False, 'import datetime\n'), ((95, 19, 95, 40), 'datetime.date.today', 'datetime.date.today', ({}, {}), '()', False, 'import datetime\n')]