"""
This code is used to scrape ScienceDirect of publication urls and write them to
a text file in the current directory for later use.
"""

import selenium
from selenium import webdriver
import numpy as np
import pandas as pd
import bs4
from bs4 import BeautifulSoup
import time
from sklearn.utils import shuffle

def scrape_page(driver):
    """
    This method finds all the publication result web elements on the webpage.
    Parameters
    ----------
    driver (Selenium webdriver object) : Instance of the webdriver class e.g.
        webdriver.Chrome()
    Returns
    -------
    elems (list) : A list of all scraped hrefs from the page
    """

    elems = driver.find_elements_by_class_name('ResultItem')
    return elems


def clean(elems):
    """
    This method takes a list of scraped selenium web elements
    and filters/ returns only the hrefs leading to publications.
    Filtering includes removing all urls with keywords that are indicative of
    non-html links.
    Parameters
    ----------
    elems (list) : The list of hrefs to be filtered
    Returns
    -------
    urls (list) : The new list of hrefs, which should be the same as the list
        displayed on gui ScienceDirect
    """

    titles = []
    urls = []
    for elem in elems:
        href_child = elem.find_element_by_css_selector('a[href]')
        url = href_child.get_attribute('href')

        title = href_child.text

        titles.append(title)
        urls.append(url)
    return urls, titles

def build_url_list(gui_prefix,search_terms,journal_list):
    """
    This method takes the list of journals and creates a tiple nested dictionary
    containing all accessible urls to each page, in each year, for each journal,
    for a given search on sciencedirect.
    """

    dict1 = {}
    years = 
np.arange(1995,2020)