Web-scraping investment fund data into Pandas
I created function get_soup because this code is offen used many times in code. In this code I could put all in get_data.

get_data gets url and uses get_soup with this url. Later it scrapes data from html, creates DataFrame and returns it

Main part uses get_data with two urls and gets two dataframes.

I put some other comments in code.

# &lt;-- remove not used modules
# &lt;-- use more popular names 
from urllib.request import urlopen
from bs4 import BeautifulSoup as BS
import pandas as pd
from IPython.display import display

# --- functions ---
# &lt;-- all functions before main part

def get_soup(url):
    # &lt;-- more readable names (and more popular)
    response = urlopen(url)
    html = response.read()
    response.close()
    soup = BS(html, "html.parser")
    return soup


def get_data(url):

    soup = get_soup(url)

    #Isolate header to get name and symbol
    h1 = soup.h1

    #Isolate stock symbol
    # &lt;-- find() to get only first item
    title = h1.find("span",{"class":"label-primary"}).text

    #print(title)

    #strip space and line break
    # &lt;-- use the same variable instead strip,strip2, strip3
    # &lt;-- maybe too much comments 
    header = h1.text.strip()

    #strip stock symbol
    header = header.strip(title)

    #strip remaining line break
    header = header.strip()

    #print(strip)

    # &lt;-- use better name 'table'
    table = soup.find_all("table",{"class":"chart base-table"})[1]

    #Create lists to fill
    #sector_data = [row.text for row in table.find_all("td")]
    sector_data = []
    # &lt;-- remove lists which will be created later

    for row in table.find_all("td"):
        sector_data.append(row.text)

    #Assign every other value to proper list to get 2 columns
    sector_name = sector_data[::2]
    sector_weight = sector_data[1::2]

    #Insert name/symbol for clarification/validation
    sector_weight.insert(0, title)
    sector_name.insert(0, header)

    # create dataframe in pandas
    # &lt;-- create DF directly with data
    df = pd.DataFrame({
       'Sector': sector_name,
       'Weight': sector_weight,
    })

    #display(df)

    return df

# --- main ---

# &lt;-- the same variable `url` because it keep the same type of data
#     and I will no need this value later - so I can resuse this name.
url = 'https://etfdb.com/etf/IWD/'
df1 = get_data(url)

url = 'https://etfdb.com/etf/IWF/'
df2 = get_data(url)

results = df1.merge(df2, on="Sector")
results.columns = ['Sector', 'IWD', 'IWF']
display(results)