# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Pick a website and describe your objective

# ### Project Outline
#
# - We're going to scrape https://www.gutenberg.org/
# - We'll get a list of famous literary books.

import urllib.request, urllib.parse, urllib.error
import requests
from bs4 import BeautifulSoup
import ssl
import re

# Use the requests library to download web pages

# - Inspect the website's HTML source and identify the right URLs to download
# - Download and save web pages locally using the requests library
# - Create a function to automate downloading for different topics/search queries

top100url = 'https://www.gutenberg.org/browse/scores/top'

response = requests.get(top100url)



# Use Beautiful Soup to parse and extract information

# - Parse and explore the structure of downloaded web pages using Beautiful Soup
# - Use the right properties and methods to extract the required information
# - Create functions to extract from the page into lists and dictionaries
# - (Optional) Use a REST API to acquire additional information

contents = response.content.decode(response.encoding)

soup = BeautifulSoup(contents, 'html.parser')


def status_check(r):
    if r.status_code==200:
        print('Success!')
        return 1
    else:
        print('Failed!')
        return -1


status_check(response)

lst_links = []
for link in soup.find_all('a'):
    lst_links.append(link.get('href'))

lst_links[:30]

lst_titles_temp=[]

start_idx = soup.text.splitlines().index('Top 100 EBooks yesterday')

for i in range(100):
    lst_titles_temp.append(soup.text.splitlines()[start_idx+2+i])

lst_titles = []
for i in range(100):
    id1,id2=re.match('^[a-zA-Z ]*',lst_titles_temp[i]).span()
    lst_titles.append(lst_titles_temp[i][id1:id2])

for l in lst_titles:
    print(l)











# Create CSV file(s) with the extracted information

# - Create functions for the end-to-end process of downloading, parsing and saving CSVs
# - Execute the function with different inputs to create a dataset of CSV files
# - Verify the information in the CSV files by reading them back using Pandas







# Document and share your work

# - Add proper headings and documentation in your Jupyter notebook
# - Publish your Jupyter notebook to your Github portfolio
# - (Optional) Write a blog post about your project and share it online



StarcoderdataJupyterScriptsDedupFiltered