# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/steve-keys/Assignment---1--Intro-to-Programming/blob/main/Copy_of_amazon_prices.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="ZdHkW9yUpSPZ"
# ## Lab 8

# + id="nAphLm9Vlzo5" colab={"base_uri": "https://localhost:8080/"} outputId="97c6a651-acbb-48ac-8976-f4aff7d589f0"
# !pip install kora -q

'''load packages'''

from bs4 import BeautifulSoup
from kora.selenium import wd

# + id="Jyp2DYzImzk-" colab={"base_uri": "https://localhost:8080/", "height": 617} outputId="53f32448-6468-43d0-87d6-0dac87b0839f"
# Establish Session
url = 'https://www.amazon.com.au'
wd.get(url)
wd

# + id="DCZLhnslm9Nm" colab={"base_uri": "https://localhost:8080/", "height": 52} outputId="95d24ed6-ca42-417f-d3ec-396794bbac65"
# Work out the URL pattern
template = 'https://www.amazon.com.au/s?k={}&rh=n%3A4913312051&ref=nb_sb_noss'

#search_term = 'ultrawide monitor'
search_term = input("What do you want to search for? ")

# replace spaces with a '+' SIGN
search_term = search_term.replace(' ','+')

url = template.format(search_term)
url

# + id="AQz3cNSMnA-E" colab={"base_uri": "https://localhost:8080/", "height": 634} outputId="67002f7f-41e4-4db1-9cbf-3e57a0823056"
search_term = input("What do you want to search for? ")
template = 'https://www.amazon.com.au/s?k={}&rh=n%3A4913312051&ref=nb_sb_noss'
search_term = search_term.replace(' ','+')
url = template.format(search_term)
wd.get(url)
wd

# + id="IFVLuMHtoBep" colab={"base_uri": "https://localhost:8080/"} outputId="213b83c0-98db-4f96-b84e-7d3a8dd25559"
search_term = input("What do you want to search for? ")
template = 'https://www.amazon.com.au/s?k={}&rh=n%3A4913312051&ref=nb_sb_noss'
search_term = search_term.replace(' ','+')
url = template.format(search_term)
wd.get(url)


# + id="pK-96OwPv-Ev"
def get_url():
  search_term = input("What do you want to search for? ")
  template = 'https://www.amazon.com.au/s?k={}&rh=n%3A4913312051&ref=nb_sb_noss'
  search_term = search_term.replace(' ','+')
  url = template.format(search_term)
  wd.get(url)


# + colab={"base_uri": "https://localhost:8080/"} id="lvggkfBVwPBv" outputId="856540c4-df24-4442-89f5-07bd27b8b309"
get_url

# + id="JTWxunzYxIwq"
soup = BeautifulSoup(wd.page_source, 'html.parser')
soup

# + [markdown] id="h4ee8kjVoYB3"
# # Extract the Collection

# + id="9FVKD7tYoaWi" colab={"base_uri": "https://localhost:8080/"} outputId="ec2a523b-e40d-4e0c-b26b-5f198d200a45"
soup = BeautifulSoup(wd.page_source, 'html.parser')
soup
results = soup.find_all('div',{'data-component-type': 's-search-result'})
results[0]

# + id="anR8Mto_olFI" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="843520c7-39b3-4d49-b1cc-94242eba465b"
results[0].h2.a.text


# + id="7SbZ48oPsNFk"
def extract_description(item):
  description = item.h2.a.text
  return description


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="2XxRgBPC_F5L" outputId="34090318-d6a5-4238-bcc3-3b193a2efcb5"
extract_description(results[10])

# + [markdown] id="1XWF5LHBsjlJ"
# # Prototype the record

# + id="cC0bk72XsQ-V" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="97c4064d-816b-40bf-c44e-934134e270b3"
item = results[0]
price_parent = item.find('span', 'a-price')
price = price_parent.find('span', 'a-offscreen')
price.text


# + id="jESVEqVVsyuA"
def extract_price(item):
  try:
    price_parent = item.find('span', 'a-price')
    price = price_parent.find('span', 'a-offscreen')
    price = price.text
  except AttributeError:
    price = ''
  return price


# + id="iO16HydCs3zv" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="9eff6c15-3520-4c85-bb8c-d537123ca0ec"
extract_price(results[2])


# + id="64GHbNF8s734"
def extract_record(item):
  return {
      'description': extract_description(item),
      'price': extract_price(item)
  }


# + id="fBB-xo7HtIsm" colab={"base_uri": "https://localhost:8080/"} outputId="28ff21d8-a271-48e0-fb28-6fadbdf4d70f"
extract_record(results[0])

# + id="4QZ_hBYRtxny" colab={"base_uri": "https://localhost:8080/"} outputId="df7d672a-62a7-48d8-d621-33748577cd82"
type(results)

# + id="LhWHC-W7t3mH" colab={"base_uri": "https://localhost:8080/"} outputId="3bdd5f49-b63a-4fe6-baef-4bee40b62c25"
for r in results:
  print(extract_record(r))

# + id="BfmPYjmRuMwU" colab={"base_uri": "https://localhost:8080/"} outputId="8f86edf6-0816-4c0f-931b-1af6e512e5f3"
records = []
for r in results:
  records.append(extract_record(r))

records[0]

# + colab={"base_uri": "https://localhost:8080/"} id="Zpema1TRfAtt" outputId="e6e90407-3627-4b96-b86c-e08104b30ebb"
wd.close()

import pandas as pd
df = pd.DataFrame.from_records(records)
df.head

# + [markdown] id="bQK8lneQvDxO"
# # Generalize the pattern

# + id="gF3DcIRRvHnP"


# + id="QqKbxj1kwcYc"
records = []
results = soup.find_all('div', {'data-component-type': 's-search-result'})

for item in results:
  records.append(extract_record(item))

# + [markdown] id="_ZRk5fnhxeRZ"
# # Error Handling

# + id="vkIT_BNUxFvl"



# + id="pYKIDlY0x64Y"


# + id="phNc9nIpyImi"


# + [markdown] id="TGU8ZMgxyaWt"
# # Find next page

# + id="7JzOCBBRyd5B"


# + [markdown] id="FanXzdQm2SHO"
# # Putting it all together

# + id="sP0evzCj2O2d"
# !pip install kora -q

'''load packages'''
from bs4 import BeautifulSoup
from kora.selenium import wd
import pandas as pd

def get_url():
  search_term = input("What do you want to search for? ")
  template = 'https://www.amazon.com.au/s?k={}&rh=n%3A4913312051&ref=nb_sb_noss'
  search_term = search_term.replace(' ','+')
  url = template.format(search_term)

  return url

def extract_description(item):
  description = item.h2.a.text
  return description
  
def extract_price(item):
  try:
    price_parent = item.find('span', 'a-price')
    price = price_parent.find('span', 'a-offscreen')
    price = price.text
  except AttributeError:
    price = ''
  return price

def extract_record(item):
  return {
      'description': extract_description(item),
      'price': extract_price(item)
  }


# + colab={"base_uri": "https://localhost:8080/", "height": 680} id="8g0sjRXfh1Rb" outputId="8902b04d-1e97-46d9-c33b-34766886e9a0"
# Establish Session
url = 'https://www.amazon.com.au'
wd.get(url)

earch_term = input("What do you want to search for today? ")
url = get_url(search_term)
wd.get(url)       
soup = BeautifulSoup(wd.page_source, 'html.parser')
records = []
results = soup.find_all('div',{'data-component-type': 's-search-result'})

for item in results:
  records.append(extract_record(item))

wd.close()
df = pd.DataFrame.from_records(records)
df.head()

# + id="faqEcoorhFb7"


# + id="SmzW9RwEASoq"
df = amonzon_as_dataframe("ultrawide monitor")

# + id="BF-zepIwAetK"


# + id="sOFfhTFDDpto"


StarcoderdataJupyterScriptsDedupFiltered