python append() and remove html tags
Question: I need some help. My output seems wrong. How can I correctly append the values
of dept, job_title, job_location. And there are html tags with the values of
dept. how can I remove those tags.

my code

    
    
    response = requests.get("http://hortonworks.com/careers/open-positions/")
    soup = BeautifulSoup(response.text, "html.parser")
    
    jobs = []
    
    
    div_main = soup.select("div#careers_list")
    
    
    for div in div_main:
        dept = div.find_all("h4", class_="department_title")
        div_career = div. find_all("div", class_="career")
        title = []
        location = []
        for dv in div_career:
            job_title = dv.find("div", class_="title").get_text().strip()
            title.append(job_title)
            job_location = dv.find("div", class_="location").get_text().strip()
            location.append(job_location)
    
        job = {
            "job_location": location,
            "job_title": title,
            "job_dept": dept
        }
        jobs.append(job)
    pprint(jobs)
    

It should look like

{'job_dept': Consulting,

'job_location':'Chicago, IL'

'job_title': Sr. Consultant - Central'

1 value for each variables.


Answer: The structure of your html is sequential, not hierarchical, so you have to
iterate through your job list and update department title as you go:

    
    
    import requests
    from bs4 import BeautifulSoup, Tag
    from pprint import pprint
    headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0'}
    response = requests.get("http://hortonworks.com/careers/open-positions/", headers=headers)
    
    soup = BeautifulSoup(response.text, "html.parser")
    
    jobs = []
    
    
    div_main = soup.select("div#careers_list")
    
    
    for div in div_main:
        department_title = ""
        for element in div:
            if isinstance(element, Tag) and "class" in element.attrs:
                if "department_title" in element.attrs["class"]:
                    department_title = element.get_text().strip()
                elif "career" in element.attrs["class"]:
                    location = element.select("div.location")[0].get_text().strip()
                    title = element.select("div.title")[0].get_text().strip()
                    job = {
                        "job_location": location,
                        "job_title": title,
                        "job_dept": department_title
                    }
                    jobs.append(job)
    
    pprint(jobs)
    

