# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="Rl4NCRZxJqci"
# ## Not exactly ***Open Source*** yunno, so no comments ✌️😌✌️

# + id="B388bjtcIOQ8"
import requests
from bs4 import BeautifulSoup
import time
import urllib

# + id="lHtIl7xLLeLn"
indeedURL = 'https://www.indeed.com/viewjob?jk=8098ca27c3755c84&tk=1f2obu3hpt48n800&from=serp&vjs=3'
cbURL = 'https://www.careerbuilder.com/job/J3T5DZ79G3K8Z5BK0X3'

# + [markdown] id="zWNeDKndM6Tz"
# ### Scrape Indeed JD

# + id="Ta7fdhlNMJVN"
indeedJDPage = requests.get(indeedURL)
indeedJDSoup = BeautifulSoup(indeedJDPage.text, "html.parser")


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="0R6HnizkNedv" outputId="baa78674-6432-4346-8c95-1a7991162ddc"
def extract_job_title_from_result(soup):
  jobTitles = []
  for div in soup.find_all(name="div", attrs={"id": "viewJobSSRRoot"}):
    for a in div.find_all(name="h1"):
      jobTitles.append(a.text.strip())
  return jobTitles[0]

extract_job_title_from_result(indeedJDSoup)


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="q_e9NCJyWxPE" outputId="0edb1fcc-1133-48ca-c392-24a22c5f2c8e"
def extract_job_location_from_result(soup):
  jobLocation = []
  for div in soup.find_all(name="div", attrs={"class": "jobsearch-DesktopStickyContainer-subtitle"}):
    for a in div.find_all(name="div"):
      jobLocation.append(a.text.strip())
  return jobLocation[1]

extract_job_location_from_result(indeedJDSoup)


# + colab={"base_uri": "https://localhost:8080/", "height": 52} id="8zhObWL8QBSC" outputId="bb2b90f3-4746-4887-831a-7a9d105439f4"
def extract_company_job_posting_from_result(soup):
  companyJobPosting = []
  for div in soup.find_all(name="div", attrs={"id": "applyButtonLinkContainer"}):
    for a in div.find_all(name="a", attrs={"role": "link"}):
      companyJobPosting.append(a['href'])
  return companyJobPosting[0]

extract_company_job_posting_from_result(indeedJDSoup)


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="C3SL9zvXVhWi" outputId="7d55dd37-4e44-4d25-f848-b894273e14db"
def extract_job_policy_from_result(soup):
  jobPolicy = []
  for div in soup.find_all(name="div", attrs={"class": "jobsearch-JobDescriptionSection-sectionItem"}):
    for a in div.find_all(name="div"):
      jobPolicy.append(a.text.strip())
  try:
    return jobPolicy[1]
  except:
    return ''

extract_job_policy_from_result(indeedJDSoup)


# + colab={"base_uri": "https://localhost:8080/"} id="Wgb49YMpWkty" outputId="c5b70d26-dd3e-4719-9804-0d82c46d8cfb"
def extract_full_jd_from_result(soup):
  fullJD = []
  fullJDString = ""
  for div in soup.find_all(name="div", attrs={"id": "jobDescriptionText"}):
    for lines in div.find_all(name="div"):
      fullJD.append(lines.text.strip())
  cache = list(dict.fromkeys(fullJD))
  for i in cache:
    if i != "" and fullJDString.find(i) < 0:
      fullJDString = fullJDString + "\n\n" + i
  return fullJDString

print(extract_full_jd_from_result(indeedJDSoup))


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="ExRPuy-ymLlX" outputId="b8eb115d-3d1c-4c3f-f3de-b254e602746a"
def extract_job_poster_and_age_from_result(soup):
  jobPosterAndAge = []
  for div in soup.find_all(name="div", attrs={"class": "jobsearch-JobMetadataFooter"}):
    for a in div.find_all(name="div"):
      jobPosterAndAge.append(a.text.strip())
  return " | ".join([jobPosterAndAge[0], jobPosterAndAge[1]])

extract_job_poster_and_age_from_result(indeedJDSoup)

# + [markdown] id="50hTXTQ3NLfN"
# ### Scrape CB JD
#

# + id="Xqb3zMQxNOkX"
cbJDPage = requests.get(cbURL)
cbJDSoup = BeautifulSoup(cbJDPage.text, "html.parser")


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="mpNHazmqqqde" outputId="8f75096e-6be1-48b3-fc3a-a33246ee22c8"
def extract_job_title_from_result(soup):
  jobTitles = []
  for div in soup.find_all(name="div", attrs={"class": "data-display"}):
    for a in soup.find_all(name="h2", attrs={"class": "h3"}):
      jobTitles.append(a.text.strip())
  return jobTitles[0]

extract_job_title_from_result(cbJDSoup)


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="0ryH16F6rnRC" outputId="6dfd1b77-21fb-4561-c2df-9c70f4e56d78"
def extract_job_location_from_result(soup):
  jobLocation = []
  for div in soup.find_all(name="div", attrs={"class": "data-display"}):
    for a in div.find_all(name="div", attrs={"class": "data-details"}):
      for b in a.find_all(name="span"):
        jobLocation.append(b.text.strip())
  return " | ".join(jobLocation)

extract_job_location_from_result(cbJDSoup)


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="XcLMcFK_tluw" outputId="f357acb4-b8fd-4379-a410-74ad5d8b6717"
def extract_company_job_posting_from_result(soup):
  companyJobPosting = []
  for div in soup.find_all(name="div", attrs={"class": "data-display"}):
    for a in div.find_all(name="a", attrs={"data-gtm": "job-action|apply-internal-top"}):
      companyJobPosting.append(a['href'])
  return companyJobPosting[0]

extract_company_job_posting_from_result(cbJDSoup)


# + colab={"base_uri": "https://localhost:8080/"} id="htqGI__2uK-s" outputId="ca1f8cfd-e1f7-4f0f-ffdf-47142848d17b"
def extract_full_jd_from_result(soup):
  fullJD = []
  fullJDString = ""
  for div in soup.find_all(name="div", attrs={"class": "jdp-description-details"}):
    for a in soup.find_all(name="div", attrs={"class": "col-mobile-full"}):
      for lines in div.find_all(name="div"):
        fullJD.append(lines.text.strip())
  cache = list(dict.fromkeys(fullJD))
  for i in cache:
    if i != "" and fullJDString.find(i) < 0 and fullJDString.find(");") < 0:
      fullJDString = fullJDString + "\n\n" + i
  return fullJDString[:-95]

print(extract_full_jd_from_result(cbJDSoup))


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="8XuCMfIkA-MQ" outputId="59237f84-399c-459d-ed8e-318b3d69c4ce"
def extract_job_salary_from_result(soup):
  jobSalaries = []
  for div in soup.find_all(name="div", attrs={"class": "data-snapshot"}):
    for a in soup.find_all(name="div", attrs={"class": "block"}):
      jobSalaries.append(a.text.strip())
  return jobSalaries[0]

extract_job_salary_from_result(cbJDSoup)

# + id="SeCpDY4mwuB2"


StarcoderdataJupyterScriptsDedupFiltered