# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
'''
    Goodreads.com scraping 
    
    Question: 
    Scraping Practice for DAT 129
        Before scraping a page check:
            1. Are the URLs encoded in a sensible way that we can use the urlib to grab the HTML easily?
            2. Are the elements on the page we want to scrape enclosed in HTML tags with distinctive class names or tage names?
'''

import urllib
from bs4 import BeautifulSoup

def get_search_url(term):
    '''assembles a query against goodreads.com given a search term'''
    url = "https://www.goodreads.com/search?query=%s" % (str(term))
    return url

def get_page_text(url):
    '''Given a URL, fetches the raw HTML'''
    #build the  request object from the given url
    req = urllib.request.Request(url)
    # access the network via the computer's standard gateway
    # to actually retrieve url from goodreads.com server
    with urllib.request.urlopen(req) as response: 
        return response.read()


# -

term = "knitting"
url = get_search_url(term)
print(url)
page_text = get_page_text(url)
print(page_text)



StarcoderdataJupyterScriptsDedupFiltered