import urllib.request
import pandas as pd
import sqlite3
import re
from bs4 import BeautifulSoup


# Parameters
postcodes_list = ["W1F7EY"]
db_name = "scraped.db"


# This is so that Deliveroo think the scraper is Google Chrome
# as opposed to a web scraper
hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11' +
       '(KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*' +
       ';q=0.8',
       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
       'Accept-Encoding': 'none',
       'Accept-Language': 'en-US,en;q=0.8',
       'Connection': 'keep-alive'}


def process_menu(doc, url, tags_df, tag_type, restaurants, restaurants_to_tags,
                 menu_sections, menu_items):
    # This function processes the menu

    # This gets the restaurant_name by finding the <h1> tag with the CSS class
    # restaurant_name
    restaurant_name = doc.find("h1", class_="restaurant__name", text=True).text

    # This gets the deliveroo_name by selecting the appropriate part from the
    # URL
    # This will fail on restaurants not in London
    deliveroo_name = re.findall(
        '(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\?postcode=)',
        url)[0]

    # This adds this to the restaurants dataframe
    # This isn't very efficient, if you were wanting to scrape large numbers
    # you wouldn't want to use .append
    restaurants = restaurants.append(
        {"name": restaurant_name, "deliveroo_name": deliveroo_name},
        ignore_index=True)

    # This gets the restaurant_id by finding the index of what as inserted
    # Again this isn't very efficient
    restaurant_id = restaurants[
        (restaurants == [restaurant_name, deliveroo_name]).all(
            axis=1)].index[0]
    restaurant_tags = []

    # Deal with tags
    # Start by finding all <small> tags with the CSS class tag
    for tag in doc.find_all("small", class_="tag"):
        # The second element of the <small> CSS class is the type of the tag
        # this could be locale or food etc.
        tagtype = tag['class'][1]
        # The name of the tag is what is inside the <small>
        name = tag.text

        # See if the tagtype exists in the tag_type dataframe
        type_matches = tag_type[(tag_type == [tagtype]).all(axis=1)]

        # If it doesn't
        if len(type_matches) == 0:
            # Add it (again not very efficient)
            tag_type = tag_type.append({"name": tagtype}, ignore_index=True)

            # Update the matches
            type_matches = tag_type[(tag_type == [tagtype]).all(axis=1)]

        # See if the tag already exists in the tags_df dataframe
        matches = tags_df[
            (tags_df == [name, type_matches.index[0]]).all(axis=1)]

        # If it doesn't
        if len(matches) == 0:
            # Add it
            entry = {"name": name, "type": type_matches.index[0]}
            tags_df = tags_df.append(entry, ignore_index=True)
            matches = tags_df[(tags_df == [name, type_matches.index[0]]).all(
                axis=1)]

        # Add the tag to a list of tags for that restaurant
        restaurant_tags.append(matches.index[0])

    # For each tag
    for tag in restaurant_tags:
        # Add this to restaurants_to_tags df
        restaurants_to_tags = restaurants_to_tags.append(
            {"restaurant_id": restaurant_id, "tag_id": tag}, ignore_index=True)

    # For each category (in the menu, e.g. Sides, Mains, Desserts, Drinks -
    # different for every restaurant though!) process the menu items
    # This is found by looking for <div> tags with the CSS class
    # menu-index-page__menu-category
    categories = doc.find_all("div", class_="menu-index-page__menu-category")
    for category in categories:
        # the category name is inside the h3 inside the div
        category_name = category.h3.text
        # Add the category to the menu_sections data frame. Again this isn't
        # efficient.
        menu_sections = menu_sections.append(
            {"restaurant_id": restaurant_id, "name": category_name},
            ignore_index=True)

        # Get the id in the menu_sections data frame
        category_id = menu_sections[
            (menu_sections == [restaurant_id, category_name]).all(
                axis=1)].index[0]

        # Get each of the items in that category
        category_items = []
        # For each menu item. Found by looking for <div> inside the category
        # with the CSS class menu-index-page__item_content
        items_html = category.find_all("div",
                                       class_="menu-index-page__item-content")
        for menu_item in items_html:
            # The name is the <h6> with the CSS class
            # menu-index-page__item-title
            item_name = \
                menu_item.find("h6", class_="menu-index-page__item-title").text

            # The price is the <span> with the CSS class
            # menu-index-page__item-price. The £ symbol is dropped, it is then
            # converted to a floating-point number (decimal), multiplied by 100
            # so that it is in pence. It is then converted to an integer.
            #
            # https://stackoverflow.com/questions/3730019/why-not-use-double-or-float-to-represent-currency
            price_as_text = \
                menu_item.find("span", class_="menu-index-page__item-price")\
                .text[1:]
            price_as_float = float(price_as_text)
            item_price = int(price_as_float * 100)

            # If an item is popular it has a <span> with the CSS class
            # menu-index-page__item-popular
            # So this tries to find it, if it exists is_item_popular = True,
            # False otherwise.
            is_item_popular = menu_item.find(
                "span", class_="menu-index-page__item-popular") is not None

            # Add this menu_item to category_items
            category_items.append(
                {"menu_section_id": category_id,
                 "name": item_name,
                 "price_in_pence": item_price,
                 "is_popular": is_item_popular}
            )

        # Add all the menu items in that category to the menu_items data frame,
        # this is more efficient than doing this one at a time
        menu_items = menu_items.append(category_items, ignore_index=True)

    # Return the updated dataframes
    return (tags_df, tag_type, restaurants, restaurants_to_tags, menu_sections,
            menu_items)


def get_restaurant_and_process_menu(url, tags_df, tag_type, restaurants,
                                    restaurants_to_tags, menu_sections,
                                    menu_items, restaurants_to_locs,
                                    postcodes):
    # This functions gets the restaurant and then processes its menu if it
    # hasn't been processed before

    # Get the deliveroo name from the url
    deliveroo_name = re.findall(
        '(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\?postcode=)',
        url)[0]

    # If this restaurant hasn't been seen before
    if deliveroo_name not in restaurants['deliveroo_name']:
        # Get the webpage
        request = urllib.request.Request(url, headers=hdr)
        page = urllib.request.urlopen(request)
        soup = BeautifulSoup(page)
        # Try and process the menu, if it doesn't work handle it nicely
        try:
            (tags_df, tag_type, restaurants, restaurants_to_tags,
             menu_sections, menu_items) = process_menu(soup, url, tags_df,
                                                       tag_type, restaurants,
                                                       restaurants_to_tags,
                                                       menu_sections,
                                                       menu_items)
        except Exception:
            print(f"Fail on {url}")

    # Get the postcode from the URL
    postcode = re.findall('(?<=\\?postcode=)(.)*', url)[0]
    # Find where it is in the postcodes data frame
    postcodes_index = (postcodes['post_code'] == postcode).index[0]

    # Find the restaurants id in the restaurants dataframe using the deliveroo
    # name
    restaurant_index = \
        (restaurants['deliveroo_name'] == deliveroo_name).index[0]

    # Add an entry to restaurants_to_locs saying that this restaurant is
    # available at this location
    restaurants_to_locs = restaurants_to_locs.append(
        {"restaurant_id": restaurant_index, "loc_id": postcodes_index},
        ignore_index=True)

    # Return the amended dataframes
    return (tags_df, tag_type, restaurants, restaurants_to_tags, menu_sections,
            menu_items, restaurants_to_locs)


def process_restaurants_for_postcode(postcode, tags_df, tag_type, restaurants,
                                     restaurants_to_tags, menu_sections,
                                     menu_items, restaurants_to_locs,
                                     postcodes):
    # This function processes the restaurants for the postcodes

    # Add the postcode to the URL - it doesn't matter that it says camden, it
    # will update as appropriate.
    url = "https://deliveroo.co.uk/restaurants/london/camden" \
        f"?postcode={postcode}&sort=time"

    # Create the HTTP request
    request = urllib.request.Request(url, headers=hdr)

    # Get the page
    page = urllib.request.urlopen(request)
    soup = BeautifulSoup(page)

    # For every link in the page
    for i, link in enumerate(soup.find_all("a")):
        print(i)
        # Get the destination of the link
        destination = link.get("href")
        # If it's to a menu, get the restaurant and process the menu
        if "/menu" in destination:
            (tags_df, tag_type, restaurants, restaurants_to_tags,
             menu_sections, menu_items, restaurants_to_locs) = \
                get_restaurant_and_process_menu(
                    "https://deliveroo.co.uk" + destination, tags_df, tag_type,
                    restaurants, restaurants_to_tags, menu_sections,
                    menu_items, restaurants_to_locs, postcodes)

    # Return the amended dataframes
    return (tags_df, tag_type, restaurants, restaurants_to_tags, menu_sections,
            menu_items, restaurants_to_locs)


def process_all_restaurants(postcodes, db_name):
    # This function processes all of the postcodes

    # Create the dataframes
    tags_df = pd.DataFrame({"name": [], "type": []})\
        .astype({"name": "str", "type": "int32"})

    tag_type = pd.DataFrame({"name": []})
    restaurants = pd.DataFrame({"name": [], "deliveroo_name": []})\
        .astype({"name": "str", "deliveroo_name": "str"})

    restaurants_to_tags = pd.DataFrame({"restaurant_id": [], "tag_id": []})\
        .astype({"restaurant_id": "int64", "tag_id": "int64"})

    menu_sections = pd.DataFrame({"restaurant_id": [], "name": []})\
        .astype({"restaurant_id": "int64", "name": "str"})

    menu_items = pd.DataFrame(
        {"menu_section_id": [],
         "name": [],
         "price_in_pence": [],
         "is_popular": []}).astype(
        {"menu_section_id": "int64",
         "name": "str",
         "price_in_pence": "int64",
         "is_popular": "bool"})

    restaurants_to_locs = pd.DataFrame({"restaurant_id": [], "loc_id": []})\
        .astype({"restaurant_id": "int64", "loc_id": "int64"})

    for post_code in postcodes['post_code']:
        (tags_df, tag_type, restaurants, restaurants_to_tags, menu_sections,
         menu_items, restaurants_to_locs) =\
            process_restaurants_for_postcode(post_code, tags_df, tag_type,
                                             restaurants, restaurants_to_tags,
                                             menu_sections, menu_items,
                                             restaurants_to_locs, postcodes)

    # Write to db
    cnx = sqlite3.connect(db_name)
    postcodes.to_sql("POSTCODES", cnx, index_label="id")
    restaurants.to_sql("RESTAURANTS", cnx, index_label="id")
    restaurants_to_locs.to_sql("RESTAURANTS_AVAILABLE", cnx, index_label="id")
    menu_items.to_sql("MENU_ITEMS", cnx, index_label="id")
    menu_sections.to_sql("MENU_SECTIONS", cnx, index_label="id")
    tags_df.to_sql("CATEGORIES", cnx, index_label="id")
    tag_type.to_sql("CATEGORY_TYPES", cnx, index_label="id")
    restaurants_to_tags.to_sql("RESTAURANT_CATEGORIES", cnx, index_label="id")
    cnx.close()


if __name__ == "__main__":
    postcodes_df = pd.DataFrame({
        'post_code': postcodes_list
    })
    process_all_restaurants(postcodes_df, db_name)

[((228, 11, 228, 30), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(228, 25, 228, 29): 'page'}, {}), '(page)', False, 'from bs4 import BeautifulSoup\n'), ((256, 15, 256, 41), 'pandas.DataFrame', 'pd.DataFrame', ({(256, 28, 256, 40): "{'name': []}"}, {}), "({'name': []})", True, 'import pandas as pd\n'), ((288, 10, 288, 34), 'sqlite3.connect', 'sqlite3.connect', ({(288, 26, 288, 33): 'db_name'}, {}), '(db_name)', False, 'import sqlite3\n'), ((301, 19, 303, 6), 'pandas.DataFrame', 'pd.DataFrame', ({(301, 32, 303, 5): "{'post_code': postcodes_list}"}, {}), "({'post_code': postcodes_list})", True, 'import pandas as pd\n'), ((36, 21, 38, 12), 're.findall', 're.findall', ({(37, 8, 37, 71): '"""(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\\\?postcode=)"""', (38, 8, 38, 11): 'url'}, {}), "('(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\\\?postcode=)', url\n    )", False, 'import re\n'), ((170, 21, 172, 12), 're.findall', 're.findall', ({(171, 8, 171, 71): '"""(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\\\?postcode=)"""', (172, 8, 172, 11): 'url'}, {}), "('(?<=https://deliveroo.co.uk/menu/london/)(.*)(?=\\\\?postcode=)', url\n    )", False, 'import re\n'), ((179, 15, 179, 34), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(179, 29, 179, 33): 'page'}, {}), '(page)', False, 'from bs4 import BeautifulSoup\n'), ((192, 15, 192, 55), 're.findall', 're.findall', ({(192, 26, 192, 49): '"""(?<=\\\\?postcode=)(.)*"""', (192, 51, 192, 54): 'url'}, {}), "('(?<=\\\\?postcode=)(.)*', url)", False, 'import re\n'), ((253, 14, 253, 52), 'pandas.DataFrame', 'pd.DataFrame', ({(253, 27, 253, 51): "{'name': [], 'type': []}"}, {}), "({'name': [], 'type': []})", True, 'import pandas as pd\n'), ((257, 18, 257, 66), 'pandas.DataFrame', 'pd.DataFrame', ({(257, 31, 257, 65): "{'name': [], 'deliveroo_name': []}"}, {}), "({'name': [], 'deliveroo_name': []})", True, 'import pandas as pd\n'), ((260, 26, 260, 75), 'pandas.DataFrame', 'pd.DataFrame', ({(260, 39, 260, 74): "{'restaurant_id': [], 'tag_id': []}"}, {}), "({'restaurant_id': [], 'tag_id': []})", True, 'import pandas as pd\n'), ((263, 20, 263, 67), 'pandas.DataFrame', 'pd.DataFrame', ({(263, 33, 263, 66): "{'restaurant_id': [], 'name': []}"}, {}), "({'restaurant_id': [], 'name': []})", True, 'import pandas as pd\n'), ((266, 17, 270, 27), 'pandas.DataFrame', 'pd.DataFrame', ({(267, 8, 270, 26): "{'menu_section_id': [], 'name': [], 'price_in_pence': [], 'is_popular': []}"}, {}), "({'menu_section_id': [], 'name': [], 'price_in_pence': [],\n    'is_popular': []})", True, 'import pandas as pd\n'), ((276, 26, 276, 75), 'pandas.DataFrame', 'pd.DataFrame', ({(276, 39, 276, 74): "{'restaurant_id': [], 'loc_id': []}"}, {}), "({'restaurant_id': [], 'loc_id': []})", True, 'import pandas as pd\n')]