# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# # Sample code for downloading articles

# ## Step 1: Prepare list of articles to download based on publishing journal

import pandas as pd

SCOPUS_1_PATH = './data/scopus_1.csv'
SCOPUS_2_PATH = './data/scopus_2.csv'
SCOPUS_3_PATH = './data/scopus_3.csv'

df_1 = pd.read_csv(SCOPUS_1_PATH)
df_2 = pd.read_csv(SCOPUS_2_PATH)
df_3 = pd.read_csv(SCOPUS_3_PATH)

full_df = pd.concat([df_1, df_2, df_3])
full_df.head()

len(full_df)

# Get list of all unique publishers
publisher_list = full_df['Publisher'].value_counts().index.tolist()
len(publisher_list)

# ## Step 2: Prepare a dataset of articles that belong to the same journals

# Create a map of journals with list of articles.
journal_dict = {}
for idx, row in full_df.iterrows():
    info = {}
    info['title'] = row['Title']
    info['doi'] = row['DOI']
    info['open-access'] = row['Open Access']
    info['issn'] = row['ISSN']

    if row['Publisher'] not in journal_dict.keys():
        journal_dict[row['Publisher']] = []

    journal_dict[row['Publisher']].append(info)

# ## Prepare list of journals from elsevier

# + id="0R-HVnZuBaTX"
elsevier_list = [i for i in publisher_list if i.startswith('Elsevier')]
elsevier_list
# -

# ## Step 3: Download all articles from elsevier journals

from time import sleep
import random
import requests
from bs4 import BeautifulSoup as soup
import urllib


def download_from_elsevier(root_path, output_filename, response_type, api_key, doi):
    """Download articles from elsevier journals

    Special Instructions: Most articles require the the download 
    request sent from an IP address inside Purdue Network. Please login to
    Purdue WiFi to run this function and get appropraite results.

    Args:
        doi (string): DOI string of requested article
        api_key (string): ELSEVIER API key to use for authentication
        response_type (string): text or html
        output_filename (): _description_
    """
    # Article url link
    url = 'http://api.elsevier.com/content/article/doi:' + doi + '?view=FULL'
    
    # HTTP headers for authentication
    # https://dev.elsevier.com/tecdoc_api_authentication.html - Read this
    headers = {
        'X-ELS-APIKEY': '{}'.format(api_key),
        'Accept': 'text/{}'.format(response_type)
    }
    
    # Make HTTP Get request
    response = requests.get(url, stream=True, headers=headers)
    
    # Save file 
    with open(root_path + output_filename + '.{}'.format(response_type), 'wb') as f:
        for chunk in response.iter_content(2048):
            f.write(chunk)


ELSEVIER_API_KEY = 'USE_YOUR_OWN_API_KEY'

# +
# Maintain information of any failed download requests
failed_articles_list = {}

# Iterate through the elsevier journal names
for idx, journal in enumerate(elsevier_list):
    article_list = journal_dict[journal]
    failed_articles_list[journal] = []
    
    # Retrieve all articles from the journal and iterate through them
    for num, article in enumerate(article_list):
        sleep(random.choice([1, 3]))
        
        doi = article['doi']
        doi = doi.replace('/', '_')
        
        # Try to download the file
        try:
            download_from_elsevier('elsevier', article['doi'],  'xml', ELSEVIER_API_KEY, "{}".format(doi))
        except Exception as e:
            print('Error while downloading from elsevier:', e)
            print('Error for article with doi {}'.format(article['doi']))
            failed_articles_list[journal].append(article)
        
        print(idx + num)
    
# -

# ## Step 4: Download articles from MDPI

mdpi_list = [i for i in publisher_list if i.startswith('MDPI')]
len(mdpi_list)


def download_from_mdpi(root_path, output_file, doi):
    """Download open access articles from MDPI journal

    Args:
        output_file (string): _description_
        doi (_type_): _description_
    """
    # MDPI Search URL link
    url = 'https://www.mdpi.com/search?q=' + urllib.parse.quote_plus(doi)
    
    # Make HTTP Get request
    response = requests.get(url)
    
    # Parse response for the actual article
    article_url = response.url
    full_text_url = article_url + '/htm'
    
    # Make HTTP Get request for the full text of the article
    resp_full_text = requests.get(full_text_url)
    
    # Save file
    with open('{}/{}.html'.format(root_path, output_file), "w", encoding='utf-8') as file:
        file.write(str(resp_full_text.text))


# Iterate through the elsevier journal names
for idx, journal in enumerate(mdpi_list):
    article_list = journal_dict[journal]
    failed_articles_list[journal] = []
    
    # Retrieve all articles from the journal and iterate through them
    for num, article in enumerate(article_list):
        sleep(random.choice([1, 3]))
        
        doi = article['doi']
        doi = doi.replace('/', '_')
        
        # Try to download the file
        try:
            download_from_mdpi(doi, "{}".format(article['doi']))
        except Exception as e:
            print('Error while downloading from MDPI:', e)
            print('Error for article with doi {}'.format(article['doi']))
            failed_articles_list[journal].append(article)
        
        print(idx + num)
    

StarcoderdataJupyterScriptsDedupFiltered