# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# Import libraries
from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
import os
import pandas as pd
pd.set_option('display.max_colwidth',1000)
import matplotlib.pyplot as plt
# %matplotlib inline
# NLTK VADER for sentiment analysis
from nltk.sentiment.vader import SentimentIntensityAnalyzer

from html_table_parser.parser import HTMLTableParser
# -

finwiz_url = 'https://finviz.com/quote.ashx?t='
finwiz_search_url = 'https://finviz.com/search.ashx?p='

# # Search for the company's ticker

# +
print('===============================================================')
search_term=str(input("Enter company name: "))#'goldman'
search_url=finwiz_search_url+search_term.upper()
print(search_url)
print('===============================================================')

req = Request(url=search_url,headers={'user-agent': 'my-app/0.0.1'})
f = urlopen(req)
xhtml = f.read().decode('utf-8')

p = HTMLTableParser()
p.feed(xhtml)
df=pd.DataFrame(p.tables[5])
df=df.rename(columns=df.iloc[0])[1:][['Ticker','Description']]
print('===============================================================')
print("Search Results for {}:\n".format(search_term))
print(df)
print('===============================================================')
print('===============================================================')
print("Find company ticker")
print('===============================================================')
# -

tickers=[]
print('===============================================================')
ticker=str(input("Enter company ticker: ")).upper()
print('===============================================================')
tickers.append(ticker)

# # Start Sentiment Analysis

# +
news_tables = {}

for ticker in tickers:
    url = finwiz_url + ticker
#     print(url)
    req = Request(url=url,headers={'user-agent': 'my-app/0.0.1'}) 
    response = urlopen(req)    
    # Read the contents of the file into 'html'
    html = BeautifulSoup(response)
    # Find 'news-table' in the Soup and load it into 'news_table'
    news_table = html.find(id='news-table')
    # Add the table to our dictionary
    if news_table:
        news_tables[ticker] = news_table
    else:
        print('No Headlines')

# +
parsed_news = []

# Iterate through the news
for file_name, news_table in news_tables.items():
    # Iterate through all tr tags in 'news_table'
    if news_table.findAll('tr'):
        for x in news_table.findAll('tr'):
            # read the text from each tr tag into text
            # get text from a only
            text = x.a.get_text() 
            # splite text in the td tag into a list 
            date_scrape = x.td.text.split()
            # if the length of 'date_scrape' is 1, load 'time' as the only element

            if len(date_scrape) == 1:
                time = date_scrape[0]

            # else load 'date' as the 1st element and 'time' as the second    
            else:
                date = date_scrape[0]
                time = date_scrape[1]
            # Extract the ticker from the file name, get the string up to the 1st '_'  
            ticker = file_name.split('_')[0]

            # Append ticker, date, time and headline as a list to the 'parsed_news' list
            parsed_news.append([ticker, date, time, text])

# Set column names
columns = ['ticker', 'date', 'time', 'headline']

# Convert the parsed_news list into a DataFrame called 'parsed_and_scored_news'
parsed_and_scored_news = pd.DataFrame(parsed_news, columns=columns)
parsed_and_scored_news.sample(5)

# +
# Instantiate the sentiment intensity analyzer
vader = SentimentIntensityAnalyzer()

# Iterate through the headlines and get the polarity scores using vader
scores = parsed_and_scored_news['headline'].apply(vader.polarity_scores).tolist()

# Convert the 'scores' list of dicts into a DataFrame
scores_df = pd.DataFrame(scores)

# Join the DataFrames of the news and the list of dicts
parsed_and_scored_news = parsed_and_scored_news.join(scores_df, rsuffix='_right')

# Convert the date column from string to datetime
parsed_and_scored_news['date'] = pd.to_datetime(parsed_and_scored_news.date).dt.date

parsed_and_scored_news.sample(5)
# -

# Set the index to ticker and date
single_day = parsed_and_scored_news.set_index(['ticker', 'date'])
# Cross-section the tsla row
single_day = single_day.xs(parsed_and_scored_news['ticker'][0])
# Select the 3rd of January of 2019
DATE=str(input("Enter Date (YYYY-MM-DD): "))
if single_day.loc[DATE].shape[0]==0:
    print('Last headline date: ',max(single_day.index).date())
else:
    single_day = single_day.loc[DATE]
    # Convert the datetime string to just the time
    single_day['time'] = pd.to_datetime(single_day['time']).dt.time
    # Set the index to time and sort by it
    single_day = single_day.set_index('time')
    # Sort it
    single_day = single_day.sort_index()


single_day

TITLE = "Negative, neutral, and positive sentiment on "+DATE
COLORS = ["red","orange", "green"]
# Drop the columns that aren't useful for the plot
plot_day = single_day.drop(['compound', 'headline'], 1)
# Change the column names to 'negative', 'positive', and 'neutral'
plot_day.columns = ['negative', 'neutral', 'positive']
# Plot a stacked bar chart
plot_day.plot.bar(stacked = True, figsize=(10, 6), title = TITLE, color = COLORS).legend(bbox_to_anchor=(1.2, 0.5))
plt.ylabel("scores");



StarcoderdataJupyterScriptsDedupFiltered