from bs4 import BeautifulSoup
import requests
import json
import datetime
import codecs
import re

featHolder = {}
featHolder['name'] = 'Pathfinder 2.0 Ancestry feat list'
featHolder['date'] = datetime.date.today().strftime("%B %d, %Y")



def get_details(link):
    res = requests.get(link)
    res.raise_for_status()
    soup = BeautifulSoup(res.text, 'lxml')
    feat = soup.find_all("div", {'class':'main'})
    detailraw = soup.find("meta", {'name':'description'})['content'] #First we grab the content from the meta tag
    detailsplit = re.split('<(.*?)>', detailraw) #Now we split it into groups of strings seperated by < >, to pull out any links
    detail = ''.join(detailsplit[::2]) #Finally, we join every other group together (passing over the link groups) into one string
    #print(detail)
    return detail
        


def get_feats(link):
    feats = []
    res = requests.get(link)
    res.raise_for_status()
    soup = BeautifulSoup(res.text, 'lxml')
    table = soup.find(lambda tag: tag.name=='table' and tag.has_attr('id') and tag['id']=="ctl00_MainContent_TableElement") 
    rows = table.findAll(lambda tag: tag.name=='tr')
    t = 0
    for row in rows:
        t += 1
        #print(row)
        #print("-----------------------------------")
        feat = {}
        entries = row.find_all(lambda tag: tag.name=='td')
        if entries is not None:
            if len(entries) > 0:
                name = entries[0].find("a").next_sibling.text #We do next_sibling here because the source puts PFS links first, which we want to skip over.
                link = entries[0].find("a").next_sibling.a['href']
                #for entry in entries: 
                #   print(entry)
                #  print("row---------------")
                level = entries[1].text
                traits = entries[2].text
                prereq = entries[3].text
                source = entries[4].text


                feat['name'] = name
                feat['level'] = level
                feat['traits'] = traits.split(", ")
                feat['link'] = "https://2e.aonprd.com/" +link
                feat['prereq'] = prereq
                feat['benefits'] = source
                details = get_details(feat['link'])
                feat['text'] = details
                feats.append(feat)
        #if t > 5:
            #break
    return feats


listOfPages = codecs.open("ancestryFeats.csv", encoding='utf-8')
for line in listOfPages: 
    featMD = line.split(",")
    print("Getting feats for :", featMD[0],"This url:", featMD[2])

    featHolder[featMD[1]] = get_feats(featMD[2].strip('\n'))

json_data = json.dumps(featHolder, indent=4)
#print(json_data)
filename = "ancestry-feats-pf2.json"
f = open(filename, "w")
f.write(json_data)
f.close

[((68, 14, 68, 64), 'codecs.open', 'codecs.open', (), '', False, 'import codecs\n'), ((75, 12, 75, 44), 'json.dumps', 'json.dumps', (), '', False, 'import json\n'), ((15, 10, 15, 28), 'requests.get', 'requests.get', ({(15, 23, 15, 27): 'link'}, {}), '(link)', False, 'import requests\n'), ((17, 11, 17, 42), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(17, 25, 17, 33): 'res.text', (17, 35, 17, 41): '"""lxml"""'}, {}), "(res.text, 'lxml')", False, 'from bs4 import BeautifulSoup\n'), ((20, 18, 20, 48), 're.split', 're.split', ({(20, 27, 20, 36): '"""<(.*?)>"""', (20, 38, 20, 47): 'detailraw'}, {}), "('<(.*?)>', detailraw)", False, 'import re\n'), ((29, 10, 29, 28), 'requests.get', 'requests.get', ({(29, 23, 29, 27): 'link'}, {}), '(link)', False, 'import requests\n'), ((31, 11, 31, 42), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(31, 25, 31, 33): 'res.text', (31, 35, 31, 41): '"""lxml"""'}, {}), "(res.text, 'lxml')", False, 'from bs4 import BeautifulSoup\n'), ((10, 21, 10, 42), 'datetime.date.today', 'datetime.date.today', ({}, {}), '()', False, 'import datetime\n')]