import requests
import time
from bs4 import BeautifulSoup


class crawl:
    """Keep track of time between scrape requests.
    args:
        wait: time between requests
        retry_max: number of times to retry
    """

    def __init__(self, wait, retry_max):
        self.wait = wait
        self.retry_max = retry_max
        self.last_scrape = time.time() - (self.wait * 0.5)
        # can let user scrape faster the first time.

    def since_scrape(self):
        return (time.time() - self.last_scrape) > self.wait

    def wait_till(self):
        while not self.since_scrape():
            time.sleep(1)

    def get(self, url):
        count = 0
        while count < self.retry_max:
            time.sleep(self.wait * count)  # sleep for successively longer times
            try:
                self.wait_till()
                response = requests.get(url)
                self.last_scrape = time.time()
                if response.status_code == requests.codes.ok:
                    return response
                else:
                    raise Exception(
                        "Non-standard issue connecting to "
                        + f"{url}: {response.status_code}."
                    )
            except requests.exceptions.RequestException as e:
                pass
            count += 1

    def get_html(self, url):
        return self.get(url).text

    def get_soup(self, url):
        return BeautifulSoup(self.get(url).text, "html.parser")

    def get_json(self, url):
        return self.get(url).json()
