import re
import httplib2
from bs4 import BeautifulSoup                    
from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo
from scraping.Constants import ENABLE_CUSTOM_QUESTIONS_FILTER, FAQ_LINKS, COLLECTION_NAME


def cleanQuestions(questions):
    questionList = []
    for question in questions:
        questionList.append(stripExtra(question.lstrip().rstrip()))

    return removeDuplicates(questionList)

def getLastAnswer(question, bodyText):
    start = bodyText.index(question) + len(question)
    text = bodyText[start : -1].lstrip()
#     print(text.lstrip())
    whitespaceCount = 0
#     print(answerLength)
    for i in range(0, len(text)):
#         print(answer[i], ' isSpace : ', answer[i].isspace())
        if text[i].isspace():
            whitespaceCount = whitespaceCount + 1
            if whitespaceCount >= 3:
#                 print(0 + i - 3)
#                 print(text[0 : 0 + i - 2])
                return text[0 : 0 + i - 2]
        else :
            if whitespaceCount != 0:
                whitespaceCount = 0

def cleanAnswer(answer):
    answerLength = len(answer)
    whitespaceCount = 0
#     print(answerLength)
    for i in range(0, answerLength):
#         print(answer[i], ' isSpace : ', answer[i].isspace())
        if answer[i].isspace():
            whitespaceCount = whitespaceCount + 1
            if whitespaceCount >= 3:
#                 print(0 + i - 3)
                return answer[0 : 0 + i - 2].lstrip()
        else :
            if whitespaceCount != 0:
                whitespaceCount = 0
    return answer.rstrip()

def getAnswers(body, questions):
    bodyText = body.getText()
#     answerTag = getAnswerTag(body, bodyText, questions)
#     print(bodyText)
    questionCount = len(questions)
    answerList = []
    for i in range(0, questionCount):
        print('Q: ', questions[i])
        if i  == questionCount - 1:
            #Last element
            answer = getLastAnswer(questions[i], bodyText)
        else :
            start = bodyText.index(questions[i]) + len(questions[i])
            end = bodyText.index(questions[i + 1], start, -1)
            print("Start : ", start , " End : ", end)
            soup1 = BeautifulSoup(bodyText[start : end], 'html.parser')
#             print(soup1)
            answer = soup1.getText().lstrip()
            
        answer = cleanAnswer(answer)
        answerList.append(answer)
        print('A: ', answer)     
    return answerList

def processWithCustomQuestions(questions):
#     isCustomQuestionsEnabled = checkConfigForFlag(ENABLE_CUSTOM_QUESTIONS_FILTER)
#     print("isCustomQuestionsEnabled : ", isCustomQuestionsEnabled)
    if ENABLE_CUSTOM_QUESTIONS_FILTER == False:
        return
    blackListedQuestions = getBlackListedQuestions()
    removeBlackListedQuestions(questions, blackListedQuestions)
    print(questions)


def getFaqOfLink(link):
#     print("LINK : ", link)
    http = httplib2.Http()
    status, html = http.request(link)
    soup = BeautifulSoup(html, 'html.parser')
    body = soup.body
    questions = cleanQuestions(soup(text=re.compile(r'\s*((?:how|How|Can|can|what|What|where|Where|describe|Describe|Who|who|When|when|Why|why|Should|should|is|Is|I|Do|do|Are|are|Will|will)[^.<>?]*?\s*\?)')))
#     print(questions)
    processWithCustomQuestions(questions)
    answerList = getAnswers(body, questions)
    return questions, answerList

# link = "https://transportation.oregonstate.edu/aabc/frequently-asked-questions"
# questions, answerList = getFaqOfLink(link)


if __name__== "__main__":
    with open(FAQ_LINKS, 'r') as myfile:
        FAQ_LINKS = myfile.read().split('\n')
      
    faqJsonList = []
    for i in range(0, len(FAQ_LINKS)):
        link = FAQ_LINKS[i]
        questions, answerList = getFaqOfLink(link)
        jsonList = convertToJsonList(link, questions, answerList)
        faqJsonList.extend(jsonList)
      
#     saveJsonToFile(faqJsonList, "output.txt")    
    saveToMongo(faqJsonList, COLLECTION_NAME)
[((13, 11, 13, 41), 'scraping.faqscrapperutil.removeDuplicates', 'removeDuplicates', ({(13, 28, 13, 40): 'questionList'}, {}), '(questionList)', False, 'from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo\n'), ((78, 27, 78, 52), 'scraping.faqscrapperutil.getBlackListedQuestions', 'getBlackListedQuestions', ({}, {}), '()', False, 'from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo\n'), ((79, 4, 79, 63), 'scraping.faqscrapperutil.removeBlackListedQuestions', 'removeBlackListedQuestions', ({(79, 31, 79, 40): 'questions', (79, 42, 79, 62): 'blackListedQuestions'}, {}), '(questions, blackListedQuestions)', False, 'from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo\n'), ((85, 11, 85, 26), 'httplib2.Http', 'httplib2.Http', ({}, {}), '()', False, 'import httplib2\n'), ((87, 11, 87, 45), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(87, 25, 87, 29): 'html', (87, 31, 87, 44): '"""html.parser"""'}, {}), "(html, 'html.parser')", False, 'from bs4 import BeautifulSoup\n'), ((111, 4, 111, 45), 'scraping.faqscrapperutil.saveToMongo', 'saveToMongo', ({(111, 16, 111, 27): 'faqJsonList', (111, 29, 111, 44): 'COLLECTION_NAME'}, {}), '(faqJsonList, COLLECTION_NAME)', False, 'from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo\n'), ((107, 19, 107, 65), 'scraping.faqscrapperutil.convertToJsonList', 'convertToJsonList', ({(107, 37, 107, 41): 'link', (107, 43, 107, 52): 'questions', (107, 54, 107, 64): 'answerList'}, {}), '(link, questions, answerList)', False, 'from scraping.faqscrapperutil import stripExtra, removeDuplicates, removeBlackListedQuestions, getBlackListedQuestions, convertToJsonList, saveToMongo\n'), ((64, 20, 64, 71), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(64, 34, 64, 55): 'bodyText[start:end]', (64, 57, 64, 70): '"""html.parser"""'}, {}), "(bodyText[start:end], 'html.parser')", False, 'from bs4 import BeautifulSoup\n'), ((89, 41, 89, 206), 're.compile', 're.compile', ({(89, 52, 89, 205): '"""\\\\s*((?:how|How|Can|can|what|What|where|Where|describe|Describe|Who|who|When|when|Why|why|Should|should|is|Is|I|Do|do|Are|are|Will|will)[^.<>?]*?\\\\s*\\\\?)"""'}, {}), "(\n    '\\\\s*((?:how|How|Can|can|what|What|where|Where|describe|Describe|Who|who|When|when|Why|why|Should|should|is|Is|I|Do|do|Are|are|Will|will)[^.<>?]*?\\\\s*\\\\?)'\n    )", False, 'import re\n')]