"""
# **Space X  Falcon 9 First Stage Landing Prediction**

"""
"""
## Web scraping Falcon 9 and Falcon Heavy Launches Records from Wikipedia

"""
"""
we will performing web scraping to collect Falcon 9 historical launch records from a Wikipedia page titled `List of Falcon 9 and Falcon Heavy launches`

https://en.wikipedia.org/wiki/List_of_Falcon\_9\_and_Falcon_Heavy_launches

"""
"""
## Objectives

Web scrap Falcon 9 launch records with `BeautifulSoup`:

*   Extract a Falcon 9 launch records HTML table from Wikipedia
*   Parse the table and convert it into a Pandas data frame

"""
import sys
import requests
# Trying to import BeautifulSoup
try:
    from bs4 import BeautifulSoup
except ImportError as e:
    !{sys.executable} -m pip install beautifulsoup4
    from bs4 import BeautifulSoup
import re
import unicodedata
import pandas as pd
def date_time(table_cells):
    """
    This function returns the data and time from the HTML  table cell
    Input: the  element of a table data cell extracts extra row
    """
    return [data_time.strip() for data_time in list(table_cells.strings)][0:2]

def booster_version(table_cells):
    """
    This function returns the booster version from the HTML  table cell 
    Input: the  element of a table data cell extracts extra row
    """
    out=''.join([booster_version for i,booster_version in enumerate( table_cells.strings) if i%2==0][0:-1])
    return out

def landing_status(table_cells):
    """
    This function returns the landing status from the HTML table cell 
    Input: the  element of a table data cell extracts extra row
    """
    out=[i for i in table_cells.strings][0]
    return out


def get_mass(table_cells):
    mass=unicodedata.normalize("NFKD", table_cells.text).strip()
    if mass:
        mass.find("kg")
        new_mass=mass[0:mass.find("kg")+2]
    else:
        new_mass=0
    return new_mass


def extract_column_from_header(row):
    """
    This function returns the landing status from the HTML table cell 
    Input: the  element of a table data cell extracts extra row
    """
    if (row.br):
        row.br.extract()
    if row.a:
        row.a.extract()
    if row.sup:
        row.sup.extract()
        
    colunm_name = ' '.join(row.contents)
    
    # Filter the digit and empty names
    if not(colunm_name.strip().isdigit()):
        colunm_name = colunm_name.strip()
        return colunm_name    

static_url = "https://en.wikipedia.org/w/index.php?title=List_of_Falcon_9_and_Falcon_Heavy_launches&oldid=1027686922"
"""
Next, request the HTML page from the above URL and get a `response` object

"""
"""
### TASK 1: Request the Falcon9 Launch Wiki page from its URL

"""
"""
First, let's perform an HTTP GET method to request the Falcon9 Launch HTML page, as an HTTP response.

"""
# use requests.get() method with the provided static_url
# assign the response to a object
page = requests.get(static_url)
page.status_code
"""
Create a `BeautifulSoup` object from the HTML `response`

"""
# Use BeautifulSoup() to create a BeautifulSoup object from a response text content
soup = BeautifulSoup(page.text, 'html.parser')
"""
Print the page title to verify if the `BeautifulSoup` object was created properly

"""
# Use soup.title attribute
soup.title
"""
### TASK 2: Extract all column/variable names from the HTML table header

"""
"""
Next, we want to collect all relevant column names from the HTML table header

"""
"""
Let's try to find all tables on the wiki page first. If you need to refresh your memory about `BeautifulSoup`, please check the external reference link towards the end of this lab

"""
# Use the find_all function in the BeautifulSoup object, with element type `table`
# Assign the result to a list called `html_tables`
html_tables = soup.find_all('table')
"""
Starting from the third table is our target table contains the actual launch records.

"""
# Let's print the third table and check its content
first_launch_table = html_tables[2]
print(first_launch_table)
"""
You should able to see the columns names embedded in the table header elements `<th>` as follows:

"""
"""
```
<tr>
<th scope="col">Flight No.
</th>
<th scope="col">Date and<br/>time (<a href="/wiki/Coordinated_Universal_Time" title="Coordinated Universal Time">UTC</a>)
</th>
<th scope="col"><a href="/wiki/List_of_Falcon_9_first-stage_boosters" title="List of Falcon 9 first-stage boosters">Version,<br/>Booster</a> <sup class="reference" id="cite_ref-booster_11-0"><a href="#cite_note-booster-11">[b]</a></sup>
</th>
<th scope="col">Launch site
</th>
<th scope="col">Payload<sup class="reference" id="cite_ref-Dragon_12-0"><a href="#cite_note-Dragon-12">[c]</a></sup>
</th>
<th scope="col">Payload mass
</th>
<th scope="col">Orbit
</th>
<th scope="col">Customer
</th>
<th scope="col">Launch<br/>outcome
</th>
<th scope="col"><a href="/wiki/Falcon_9_first-stage_landing_tests" title="Falcon 9 first-stage landing tests">Booster<br/>landing</a>
</th></tr>
```

"""
"""
Next, we just need to iterate through the `<th>` elements and apply the provided `extract_column_from_header()` to extract column name one by one

"""
column_names = []
temp = soup.find_all('th')
for x in range(len(temp)):
    try:
     name = extract_column_from_header(temp[x])
     if (name is not None and len(name) > 0):
        column_names.append(name) 
    except:
     pass
"""
Check the extracted column names

"""
print(column_names)
"""
## TASK 3: Create a data frame by parsing the launch HTML tables

"""
"""
We will create an empty dictionary with keys from the extracted column names in the previous task. Later, this dictionary will be converted into a Pandas dataframe

"""
launch_dict= dict.fromkeys(column_names)

# Remove an irrelvant column
del launch_dict['Date and time ( )']


launch_dict['Flight No.'] = []
launch_dict['Launch site'] = []
launch_dict['Payload'] = []
launch_dict['Payload mass'] = []
launch_dict['Orbit'] = []
launch_dict['Customer'] = []
launch_dict['Launch outcome'] = []
launch_dict['Version Booster']=[]
launch_dict['Booster landing']=[]
launch_dict['Date']=[]
launch_dict['Time']=[]
"""
Next, we just need to fill up the `launch_dict` with launch records extracted from table rows.

"""
"""
Usually, HTML tables in Wiki pages are likely to contain unexpected annotations and other types of noises, such as reference links `B0004.1[8]`, missing values `N/A [e]`, inconsistent formatting, etc.

"""
extracted_row = 0
#Extract each table 
for table_number,table in enumerate(soup.find_all('table',"wikitable plainrowheaders collapsible")):
   # get table row 
    for rows in table.find_all("tr"):
        #check to see if first table heading is as number corresponding to launch a number 
        if rows.th:
            if rows.th.string:
                flight_number=rows.th.string.strip()
                flag=flight_number.isdigit()
        else:
            flag=False
        #get table element 
        row=rows.find_all('td')
        #if it is number save cells in a dictonary 
        if flag:
            extracted_row += 1
            # Flight Number value
      
            launch_dict["Flight No."].append(flight_number)

            datatimelist=date_time(row[0])
            # Date value
        
            date = datatimelist[0].strip(',')
            launch_dict["Date"].append(date)
    
            
            # Time value
        
            time = datatimelist[1]
            launch_dict["Time"].append(time)

              
            # Booster version

            bv=booster_version(row[1])
            if not(bv):
                bv=row[1].a.string
            launch_dict["Version Booster"].append(bv)
 
            
            # Launch Site
       
            launch_site = row[2].a.string
            launch_dict["Launch site"].append(launch_site)

            
            # Payload
  
            payload = row[3].a.string
            launch_dict["Payload"].append(payload)

            
            # Payload Mass
         
            payload_mass = get_mass(row[4])
            launch_dict["Payload mass"].append(payload_mass)
  
            
            # Orbit

            orbit = row[5].a.string
            launch_dict["Orbit"].append(orbit)
            
            # Customer
       
            customer = row[6].a.string
            launch_dict["Customer"].append(customer)

            # Launch outcome
       
            launch_outcome = list(row[7].strings)[0]
            launch_dict["Launch outcome"].append(launch_outcome)

            
            # Booster landing
        
            booster_landing = landing_status(row[8])
            launch_dict["Booster landing"].append(booster_landing)

headings = []
for key,values in dict(launch_dict).items():
    if key not in headings:
        headings.append(key)
    if values is None:
        del launch_dict[key]

def pad_dict_list(dict_list, padel):
    lmax = 0
    for lname in dict_list.keys():
        lmax = max(lmax, len(dict_list[lname]))
    for lname in dict_list.keys():
        ll = len(dict_list[lname])
        if  ll < lmax:
            dict_list[lname] += [padel] * (lmax - ll)
    return dict_list

pad_dict_list(launch_dict,0)

df = pd.DataFrame.from_dict(launch_dict)
df.head()
df.to_csv('spacex_web_scraped.csv', index=False)
"""
#### Author : Ashlin Darius Govindasamy
"""