# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import requests
import re
import pandas as pd
from bs4 import BeautifulSoup

page = requests.get('https://en.wikipedia.org/wiki/Fantastic_Beasts_and_Where_to_Find_Them_(film)')
soup = BeautifulSoup(page.content, 'html.parser')
li = soup.find('li', class_='interlanguage-link interwiki-zh')
lin = li.find('a')
link = lin['href']

df = pd.read_excel('titles2018.xlsx')
df.head()

titles = df['Titles']
type(titles)

len(titles)

links = []

for idx, title in enumerate(titles):
    title = title.replace(' ','_')
    print('Now processing title number' + str(idx))
    try:
        try:
            tmp = []
            path = 'https://en.wikipedia.org/wiki/' + title + '_(film)'
            page = requests.get(path)
            soup = BeautifulSoup(page.content, 'html.parser')
            li = soup.find('li', class_='interlanguage-link interwiki-zh')
            lin = li.find('a')
            link = lin['href']
            tmp.append(link)
            print(len(tmp))
            links.append(tmp)
        except:
            tmp = []
            path = 'https://en.wikipedia.org/wiki/' + title
            page = requests.get(path)
            soup = BeautifulSoup(page.content, 'html.parser')
            li = soup.find('li', class_='interlanguage-link interwiki-zh')
            lin = li.find('a')
            link = lin['href']
            tmp.append(link)
            print(len(tmp))
            links.append(tmp)
    except:
        links.append('404')
        print('Not Found.')

len(links)

# Save links to csv
links = pd.DataFrame(data={"links": links})
links.to_csv("links.csv", sep=',')

# +
# Create a Pandas Excel writer using XlsxWriter as the engine.
writer = pd.ExcelWriter('links.xlsx', engine='xlsxwriter')

# Convert the dataframe to an XlsxWriter Excel object.
links.to_excel(writer, sheet_name='links')

# Close the Pandas Excel writer and output the Excel file.
writer.save()

StarcoderdataJupyterScriptsDedupFiltered