from bs4 import BeautifulSoup
import logging
import pandas as pd
import csv
import re
import requests

from urllib.parse import urljoin

logging.basicConfig(format="%(asctime)s %(levelname)s:%(message)s", level=logging.INFO)


def get_html(url):
    return requests.get(url).text


class SenateCrawler:
    def __init__(self):
        self.base_url = "https://www25.senado.leg.br/"
        self.search_url = self.base_url + "web/senadores/em-exercicio/-/e/por-nome"
        self.senate = []

    def get_senate(self, url):
        soup = BeautifulSoup(get_html(self.search_url), "html.parser")
        trs = soup.find("table").find("tbody").find_all("tr")
        for tr in trs:
            cells = tr.find_all("td")
            senateperson = {
                "name": cells[0].get_text(),
                "party": cells[1].get_text(),
                "email": cells[5].get_text(),
            }

            if senateperson["email"]:
                self.senate.append(senateperson)

    def run(self):
        try:
            self.get_senate(self.search_url)
        except Exception:
            logging.exception("global failure")
        finally:
            df = pd.DataFrame(self.senate)
            df.to_csv("senate.csv")
            logging.info("program exited")

[((137, 229), 'logging.basicConfig', 'logging.basicConfig', ([], {'format': '"""%(asctime)s %(levelname)s:%(message)s"""', 'level': 'logging.INFO'}), "(format='%(asctime)s %(levelname)s:%(message)s', level=\n    logging.INFO)\n", (156, 229), False, 'import logging\n'), ((257, 274), 'requests.get', 'requests.get', (['url'], {}), '(url)\n', (269, 274), False, 'import requests\n'), ((1171, 1196), 'pandas.DataFrame', 'pd.DataFrame', (['self.senate'], {}), '(self.senate)\n', (1183, 1196), True, 'import pandas as pd\n'), ((1245, 1275), 'logging.info', 'logging.info', (['"""program exited"""'], {}), "('program exited')\n", (1257, 1275), False, 'import logging\n'), ((1101, 1136), 'logging.exception', 'logging.exception', (['"""global failure"""'], {}), "('global failure')\n", (1118, 1136), False, 'import logging\n')]