# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as ureq
from selenium import webdriver
import time
import re

url = "https://www.uoguelph.ca/engineering/mechanical-engineering-restricted-electives-20202021"

# +
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--incognito')
#chrome_options.add_argument('--headless')

driver = webdriver.Chrome("C:\\Users\\jerry\\Downloads\\chromedriver", options=chrome_options)
# -

driver.get(url)

# # 1. Collect course link texts for driver to click on

page_soup = soup(driver.page_source, 'lxml')

containers = page_soup.find("div", {"class": "field-item even"}).findAll("a")

link_texts = [container.text for container in containers if bool(re.match("ENGG\*[0-9]{4}", container.text))]
len(link_texts)

link_texts

# # 2. Scrape courses

driver.find_element_by_id("becookiebarbuttonid").click()

driver.find_element_by_link_text(link_texts[0]).click() #go to page listing all engg courses. Scrape from there

page_soup = soup(driver.page_source, 'lxml')

containers = page_soup.findAll("div", {"class": "courseblock"})

# Turns out all courses will have to be found and scraped in one webpage listing all engg courses. So that's what the script below will do, using the link texts collected earlier to find the target courses

# +
counter = 0
course_names = []
course_codes = []
course_descs = []

for link_text in link_texts:
    for container in containers:
        if container.find("span", {"class": "text detail-code margin--small text--semibold text--big"}).text.strip() == link_text.strip():
            course_codes.append(container.find("span", {"class": "text detail-code margin--small text--semibold text--big"}).text.strip())
            course_names.append(container.find("span", {"class": "text detail-title margin--small text--semibold text--big"}).text.strip())
            course_descs.append(container.find("p", {"class": "courseblockextra noindent"}).text)
            
            counter += 1
            continue
            
print("scraped {} courses".format(counter))
# -

# # 3. Inspect and write to CSV

course_codes

course_names

course_descs

# +
import pandas as pd

df = pd.DataFrame({
    "Course Number": course_codes,
    "Course Name": course_names,
    "Course Description": course_descs    
})

df
# -

df.to_csv('UGuelph_MechEng_Technical_Electives_Courses.csv', index = False)

driver.quit()

StarcoderdataJupyterScriptsDedupFiltered