!pip3 install bs4

from bs4 import BeautifulSoup

from urllib.request import urlopen
html_str = urlopen("http://static.decontextualize.com/kittens.html").read()

print(html_str)

document = BeautifulSoup(html_str,"html.parser")

type(document)

h1_tag = document.find('h1')

h1_tag.string

img_tag = document.find('img')

img_tag.string

img_tag('src')

img_tag['src']

document.find_all('img')

img_tags=document.find_all('img')

type(img_tags)

first_img = img_tags[0]

first_img['src']

second_img = img_tags[1]

second_img['src']

for item in img_tags:
    print(item['src'])

h2_tags = document.find_all('h2')
for item in h2_tags:
    print(item.string)

checkups = document.find_all('span',{'class':'lastcheckup'})
for item in checkups:
    print(item.string)

kittens = document.find_all('div', {'class': 'kitten'})
for item in kittens:
    h2_tag = item.find('h2')
    print(h2_tag.string)
    checkup = item.find('span')
    print(checkup.string)

kittens = document.find_all('div', {'class': 'kitten'})

first_kitten = kittens[0]
first_kitten_h2 = first_kitten.find('h2')
print(first_kitten_h2.string)

planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]

separator = ","

separator.join(planets)

"""
Explanation: Scrapin' the Webz
End of explanation
"""


print("&\n".join(planets))

print("&\n".join(planets[:4]))

kittens = document.find_all('div', {'class': 'kitten'})
for item in kittens:
    h2_tag = item.find('h2')
    print(h2_tag.string)
    a_tags = item.find_all('a') #anchor tag, ancestor
    all_shows_str = [] #create a new list
    for a_tag_item in a_tags:
        #print("-", a_tag_item.string)
        tag_str = a_tag_item.string
        all_shows_str.append(tag_str)
        string_with_all_show_names = ",".join(all_shows_str)  
        print(h2_tag.string+ ":", string_with_all_show_names)
       
  

kittens_data = list()#create an empty list
kittens = document.find_all('div', {'class': 'kitten'})
for item in kittens:
    h2_tag = item.find('h2')
    print(h2_tag.string)
    a_tags = item.find_all('a') #anchor tag, ancestor
    all_shows_str = [] #create a new list
    for a_tag_item in a_tags:
        #print("-", a_tag_item.string)
        tag_str = a_tag_item.string
        all_shows_str.append(tag_str)
        #1 create a dictionary and add to it the relevant key/value pairs
        #kitten_map = {}
        #kitten_map["name"] = h2_tag.string
        #kitten_map["tvshows"] = all_shows_str
        kitten_map = {"name":h2_tag.string, "tvshows":all_shows_str }
        #2 append that dictionary to the kittens_data
        string_with_all_show_names = ",".join(all_shows_str)  
        #print(h2_tag.string+ ":", string_with_all_show_names
kittens_data

kittens_data = list()#create an empty list
kittens = document.find_all('div', {'class': 'kitten'})
for item in kittens:
    h2_tag = item.find('h2')
    print(h2_tag.string)
    a_tags = item.find_all('a') #anchor tag, ancestor
    all_shows_str = []
    for a_tag_item in a_tags:
        tag_str = a_tag_item.string
        all_shows_str.append(tag_str)
    #create a dictionary adding kittens checkups
    checkup = item.find('span')# get the string with checkup.string
    kittens_data.append(
              {"name":h2_tag.string, 
                        "tvshows":all_shows_str,
                        "last_checkup": checkup.string})
kittens_data

"""
Explanation: But first, an aside about joining strings
End of explanation
"""


Our next goal is to create a data structure that looks like this: 
    [
        {'name': 'Fluffy',
          'tv shows': ['Deep Space Nine', 'Mr.Belvedere']},
        {}

x = ["a", "b", "c", "d"]

x[0]

x.append("e")

len(x)

x[4]

numbers = [1,2,3,4,5,6]
# end up with: [1,4,9,16,25,36]

squared = [item * item for item in numbers]
for item in numbers:
    s = item*item
    squared.append(s)

squared

## Aside the Third: Making dictionaries
#declaring a dictionary
x = {'a':1, 'b':2, 'c':3}

#get a value out of a dictionary
x['a']

x.keys()

for key in x.keys():
    print(key) #print out keys

# target: {1:1, 2:4, 3:9, 4:16, 5:25,...}
squares = {}
for n in range(1,11):
    squares[n] = n*n
squares

squares[7]

names = ["Aaron", "Bob", "Caroline", "Daphne"]
#target: {"Aaron": 5} #show the name and how many characters each name has
name_length_map = {}#map is 
for item in names:
    name_length_map[item] = len(item)
name_length_map #evaluate the dictionary, Python 3 # take a list and create a new dictionary


"""
Explanation: Another Aside: lists and ...lists
End of explanation
"""


from urllib.request import urlopen
faculty_html = urlopen("http://www.journalism.columbia.edu/page/10/10?category_ids%5B%5D=2&category_ids%5B%5D=3&category_ids%5B%5D=37").read()

document = BeautifulSoup(faculty_html, "html.parser")

document.find('h2').string

h2_tag = document.find('h2')
h2_tag.string

"""
Explanation: Scraping the Faculty, how many percentage of the CJ faculty are adjunct faculty
our hypothesis:
find all the <li> tags
inside the <li> tags, find the <h4>
inside the <h4>, the name is the content of an <a>
inside the <li> tags, find the <p> with class description
** the title of the professor is the content of that tag.
End of explanation
"""


# this doesn't work, 
ul_tag = document.find('ul', {'class': 'experts-list'})
li_tags= ul_tag.find_all('li')
for item in li_tags:
    h4_tag = item.find('h4')
    if h4_tag: #none counts as false in python, only proceed if we actually found a h4-tag under li tags
        a_tag = h4_tag.find('a')#name of adjunct
        p_tag = item.find('p', {'class':'description'})#position of adjunct
        print(a_tag.string, "/", p_tag.string)

"""
Explanation: very first task: print out the names of all the faculty members.
End of explanation
"""


profs = []
ul_tag = document.find('ul', {'class': 'experts-list'})
li_tags= ul_tag.find_all('li')
for item in li_tags:
    h4_tag = item.find('h4')
    if h4_tag: #none counts as false in python, only proceed if we actually found a h4-tag under li tags
        a_tag = h4_tag.find('a')
        p_tag = item.find('p', {'class':'description'})
        prof_map = {'name': a_tag.string, 'title': p_tag.string}
        profs.append(prof_map)
profs

for item in profs:
    print(item['name'])

"""
Explanation: Now, we want to make a list of dictionaries of faculty members along with their titles
[{'name': 'Bodarky George', 'title': 'Adjunct Assistant Professor '},
 {'name':''}]
End of explanation
"""


# print all of the professors whose last name start with 'M'
m_profs = []
mcount = 0
for item in profs:
    prof_name = item['name']
    if prof_name[0]=='M':
        print(item['name'])
        mcount += 1 #mcount= mcount+1
        
print(mcount)



# find all of the professors listed as "Adjunct Faculty"
adjunct_profs = []
#same as where clause
mcount=0
for item in profs:
    if item['title'] is not None and ("Adjunct" in item['title']):
        adjunct_profs.append(item)
len(adjunct_profs)


for item in profs:
    if item['title'] is not None and ("Adjunct" in item['title']: adjunct_profs.append(item)

message = "bungalow"
message[0]

message[2:6]

message[-1]

message[0:3]

message[:3]

message[4:]

message[-5:-2]

"""
Explanation: String Indexing
print all of the professors whose last name start with M
End of explanation
"""


x=5

x

x = x-1

x

x -= 1

x

x *=2

x

"""
Explanation: lost count of asides
End of explanation
"""


