__author__ = "<NAME>"
__license__ = "Apache 2"
__version__ = "1.0.0"
__maintainer__ = "<NAME>"
__website__ = "https://llp.berkeley.edu/asgari/"
__git__ = "https://github.com/ehsanasgari/"
__email__ = "<EMAIL>"
__project__ = "1000Langs -- Super parallel project at CIS LMU"

#!/usr/bin/env python3
"""Crawl bibles hosted on http://pngscriptures.org."""
'''
This code is largely inspired/adapted from Michael Cysouw's crawling code
'''

import sys
sys.path.append('../')
import os.path
import sys
import time
import urllib
from urllib.parse import urljoin, urlsplit
from utility.file_utility import FileUtility

import requests
from lxml import html, etree


class BibleCrawler(object):
    SLEEPTIME = 0  # seconds
    log = []

    def run_crawler(self, nextpath, url, destination_directory, website='generic'):
        '''
        :param nextpath:
        :param url:
        :param destination_directory:
        :param website:
        :return:
        '''
        self.url=url
        self.nextpath = nextpath
        self.website = website
        self.counter=-1
        session = requests.Session()
        session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0',
                                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                                'Accept-Language': 'en-US,en;q=0.5'})
        if self.print:
            print(time.strftime('%H:%M:%S'), url, file=sys.stderr)
        self.seen = set()
        self.useless_url = set()
        flag=True
        count=0
        while flag and count <2000:
            if (url in self.seen and  not self.website == 'PNG') or (self.website == 'PNG' and self.counter>=1188):
                if self.print:
                    print('Break on seen url:', url, file=sys.stderr)
                BibleCrawler.log.append('\t'.join(['Break on seen url:', str(url)]))
                flag=False
                break
            self.seen.add(url)
            if self.print:
                print(url)
            for i in range(3):
                try:
                    response = session.get(url)
                    break
                except Exception as e:
                    if self.print:
                        print("Unable to reach: ", url, " : ", e)
                    return
            if response.status_code != requests.codes.ok:
                if self.print:
                    print('Error', url, response.url, response.status_code, file=sys.stderr)
                    print(time.strftime('%H:%M:%S'), url, file=sys.stderr)
                BibleCrawler.log.append(
                    '\t'.join([ 'Error', str(url), str(response.url), str(response.status_code)]))
                if self.website == 'PNG':
                    url=self.jump_url()
                    if not url:
                        flag=False
                        return
                else:
                    flag=False
                    return
            self.save_response(response, destination_directory)
            url = self.get_next_url(response)
            if not url or not url.startswith('http'):
                if self.print:
                    print('Break on invalid url:', url, file=sys.stderr)
                BibleCrawler.log.append('\t'.join(['Break on invalid url:', str(url)]))
                if self.website == 'PNG' and self.counter>=1188:
                    url=self.jump_url()
                else:
                    flag=False
                    break
            if BibleCrawler.SLEEPTIME>0:
                time.sleep(BibleCrawler.SLEEPTIME)
            count+=1
        if self.print:
            print(time.strftime('%H:%M:%S'), url, file=sys.stderr)

    def jump_url(self):
        '''
        :return:
        '''
        while self.counter < 1188:
            self.counter+=1
            url_select='/'.join(self.url.split('/')[0:-1])+'/'+FileUtility.load_list('../meta/pngscript_filenames.txt')[self.counter]
            if url_select not in self.seen and url_select not in self.useless_url:
                if requests.get(url_select).status_code==404:
                    if requests.get('/'.join(self.url.split('/')[0:-1])).status_code==404:
                        self.counter=1189
                        return None
                    self.useless_url.add(url_select)
                else:
                    url=url_select
                    self.useless_url.add(url)
                    return url
        return None

    def get_filename(self, url, base_dir):
        '''
        :param url:
        :param base_dir:
        :return:
        '''
        """Derive a filename from the given URL"""
        parts = urlsplit(url)
        path_parts = parts.path.split('/')
        if path_parts[-1] == '':
            path_parts.pop()
            path_parts[-1] += '.html'
        dir_name = os.path.join(base_dir, *path_parts[1:-1])
        if not os.access(dir_name, os.F_OK):
            os.makedirs(dir_name)
        filename = os.path.join(dir_name, path_parts[-1])
        return filename

    def save_response(self, response, base_dir):
        '''
        :param response:
        :param base_dir:
        :return:
        '''

        filename = self.get_filename(response.url, base_dir)

        if self.website == 'JW':
            # try to save only a part of the response
            tree = html.fromstring(response.content)
            text_divs = tree.xpath('//div[@id="bibleText"]')
            text_div = text_divs[0] if text_divs else None
            if text_div is not None:
                with open(filename, 'wb') as f:
                    f.write(etree.tostring(text_div))
            else:
                with open(filename, 'wb') as f:
                    f.write(handle.write(response.content))
        elif self.website == "bible.com":
            # try to save only a part of the response
            tree = html.fromstring(response.content)
            text_divs = tree.xpath('//div[contains(@class,"yv-bible-text")]')
            text_div = text_divs[0] if text_divs else None
            if text_div is not None:
                with open(filename, 'wb') as f:
                    f.write(etree.tostring(text_div))
            else:
                with open(filename, 'wb') as f:
                    f.write(handle.write(response.content))
        else:
            with open(filename, 'wb') as f:
                f.write(response.content)

    def get_next_url(self, response):
        tree = html.fromstring(response.content)
        if self.website == 'JW':
            link = tree.xpath('//div[@class="navLinkNext"]/a/@href')
            if link:
                url = urllib.parse.urljoin(response.url, link[0])
            else:
                url = None
            return url
        elif self.website == 'bible.com' or self.website=='bible.org':
            xpath_result = tree.xpath(
                '//a[contains(@class, "bible-nav-button nav-right fixed dim br-100 ba b--black-20 pa2 pa3-m flex items-center justify-center bg-white right-1")]//@href')
            relevant = xpath_result[0] if len(xpath_result) >= 1 else None
            # mydivs = soup.findAll("a", {"class": 'bible-nav-button nav-right fixed dim br-100 ba b--black-20 pa2 pa3-m flex items-center justify-center bg-white right-1'})
            # print ('Yes', 'bible-nav-button nav-right fixed dim br-100 ba b--black-20 pa2 pa3-m flex items-center justify-center bg-white right-1' in str(response.content))
        elif self.website == 'generic' or self.website == 'PNG':
            xpath_result = list(set(tree.xpath(self.nextpath)))
            relevant = xpath_result[0] if len(xpath_result) == 1 else None
        if relevant:
            return urljoin(response.url, relevant)
        else:
            return None
