text: # -*- coding: utf-8 -*-
  """Predict.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1pA3KZ0O0jgCHyz4AVnuOvtwecBJVAmmw
  """
  
  !pip3 install yfinance
  !pip3 install tweepy
  !pip3 install snscrape
  
  # from google.colab import drive
  # drive.mount('/content/gdrive')
  
  # get_tweets('NVAX')
  !pip3 install xgboost
  import xgboost
  xgboost.__version__
  
  from datetime import date,timedelta
  import json
  import csv
  import tweepy
  import re
  from datetime import datetime
  
  from urllib.request import urlopen, Request
  from bs4 import BeautifulSoup
  import os
  import pandas as pd
  import matplotlib.pyplot as plt
  from nltk.sentiment.vader import SentimentIntensityAnalyzer
  import pickle
  import yfinance as yf
  import numpy as np
  
  # from google.colab import drive
  # drive.mount('/content/gdrive')
  
  # 1. Retrieve Historical Stock Data:
  def get_stock(ticker):
      print('Step1')
      end_date=date.today()+timedelta(days=1)
      start_date=date.today()-timedelta(days=1)
      # end_date = '2021-04-14'
      # start_date = '2021-02-16'
      # 1.1 Request data:
      stock_data = yf.download(ticker,
                        start=start_date,
                        end=end_date,
                        interval='30m',
                        progress=False)
      # 1.2 Feature Engineering:
      stock_data['Percent Price Change Within Period'] = ((stock_data['Close'] - stock_data['Open'])/stock_data['Open'])*100
      stock_data['Scaled Volume'] = stock_data['Volume']/stock_data['Volume'].mean()
      data_SMA = stock_data['Adj Close'].rolling(window=3).mean().shift(1)
      stock_data['SMA(3)'] = data_SMA
      stock_data.drop(['Open','High','Low','Close'],axis=1,inplace=True)
      stock_data.reset_index(inplace=True)
      stock_data['Datetime']=stock_data['Datetime'].dt.tz_convert('America/Montreal').dt.tz_localize(None)
      return stock_data
  
  # 2. Retrieve Headlines:
  def get_news(ticker_code):
      print('Step2')
  # 2.1 Define URL:
      finwiz_url = 'https://finviz.com/quote.ashx?t='
  # 2.2 Requesting data:
      news_tables = {}
      tickers = [ticker_code]
      for ticker in tickers:
          url = finwiz_url + ticker
          req = Request(url=url,headers={'user-agent': 'my-app/0.0.1'})
          response = urlopen(req)
          html = BeautifulSoup(response,features="lxml")
          news_table = html.find(id='news-table')
          news_tables[ticker] = news_table
  # 2.3 Parsing news:
      parsed_news = []
  
      for file_name, news_table in news_tables.items():
          for x in news_table.findAll('tr'):
              text = x.a.get_text()
              date_scrape = x.td.text.split()
              if len(date_scrape) == 1:
                  time = date_scrape[0]
              else:
                  date = date_scrape[0]
                  time = date_scrape[1]
              ticker = file_name.split('_')[0]
              parsed_news.append([ticker, date, time, text])
  # 2.4 Split into columns and save:
      vader = SentimentIntensityAnalyzer()
      columns = ['ticker', 'date', 'time', 'headline']
      parsed_and_scored_news = pd.DataFrame(parsed_news, columns=columns)
      scores = parsed_and_scored_news['headline'].apply(vader.polarity_scores).tolist()
      scores_df = pd.DataFrame(scores)
      parsed_and_scored_news = parsed_and_scored_news.join(scores_df, rsuffix='_right')
      parsed_and_scored_news.insert(loc=1, column='timestamp', value=(pd.to_datetime(parsed_and_scored_news['date'] + ' ' + parsed_and_scored_news['time'])))
      parsed_and_scored_news.drop(columns=['date','time'],axis=1,inplace=True)
      return parsed_and_scored_news
  
  # 3. Define Preprocessing Functions:
  def remove_pattern(input_txt, pattern):
      print('Step3')
      r = re.findall(pattern, input_txt)
      for i in r:
          input_txt = re.sub(i, '', input_txt)
      return input_txt
  
  def clean_tweets(tweets):
      tweets = np.vectorize(remove_pattern)(tweets, "RT @[\w]*:")
      tweets = np.vectorize(remove_pattern)(tweets, "@[\w]*")
      tweets = np.vectorize(remove_pattern)(tweets, "https?://[A-Za-z0-9./]*")
      tweets = np.core.defchararray.replace(tweets, "[^a-zA-Z]", " ")
      return tweets
  
  # CONSUMER_KEY    = '3jmA1BqasLHfItBXj3KnAIGFB'
  # CONSUMER_SECRET = 'imyEeVTctFZuK62QHmL1I0AUAMudg5HKJDfkx0oR7oFbFinbvA'
  # ACCESS_TOKEN  = '265857263-pF1DRxgIcxUbxEEFtLwLODPzD3aMl6d4zOKlMnme'
  # ACCESS_TOKEN_SECRET = 'uUFoOOGeNJfOYD3atlcmPtaxxniXxQzAU4ESJLopA1lbC'
  #
  # consumer_key = '3ojILK8uar5EzGhw7pi8btQgy'
  # consumer_secret = 'lbLtusxze6qNPBlh3dIgDyVJpe3s4ILeFb3Zok6JBNqHDUqFAg'
  # access_token = '805083048626659328-eHqUuSWJPoXt478JxLcylFmKK6hTyR1'
  # access_token_secret = 'ccIy78FXeIYAtmnFjCqQTVisgTSrQR4c6TpJcY034JYpf'
  #
  # consumer_key="wFbNAH9knvKdzAu7bCppJ06bP"
  # consumer_secret="upJoW4B6HzgxWpl8QCW3qIZvcC1ejgnQbZDCF1d3ID5w9okpEh"
  # access_token_key="895520206428053505-erMLtAdiqYtNq2zlA2L8xOepvWywsdv"
  # access_token_secret="nlQkkHIHwaWbOREvg8UJHDtHb6XTLkCJDmBV1o16RcEpd"
  
  # -*- coding: utf-8 -*-
  """Prediction.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1QTyrf1Wz6Zdo7FHudRY_I53-3IKm0-XC
  """
  
  
  
  import snscrape.modules.twitter as sntwitter
  import pandas as pd
  import random
  
  def get_count(x):
    try:
      followers_count = api.get_user(x).followers_count
      print(followers_count)
      return followers_count
    except:
      return -1
  
  # 4. Retrieve Tweets:
  def get_tweets(hashtag_phrase):
      print('Step4')
      format_hashtag = hashtag_phrase
  
      scraped_tweets = sntwitter.TwitterHashtagScraper(format_hashtag).get_items()
      scraped_tweets = pd.DataFrame(scraped_tweets)[['date', 'content']]
      # scraped_tweets['followers_count'] = random.randint(1000, 100000)
  
      twitter_posts = pd.DataFrame(columns=['timestamp', 'tweet_text', 'followers_count'])
      twitter_posts['timestamp']=scraped_tweets['date']
      twitter_posts['tweet_text']= scraped_tweets['content']
      twitter_posts['followers_count']= random.randint(1000, 10000)
      twitter_posts['tweet_text']= twitter_posts['tweet_text']
      twitter_posts['scaled_followers_count'] =twitter_posts['followers_count']/twitter_posts['followers_count'].max()
      # 4.1 Feature Engineering: Sentiment Analysis
      vader = SentimentIntensityAnalyzer()
      print(twitter_posts)
      #twitter_posts['tweet_text'] = clean_tweets(twitter_posts['tweet_text'])
      scores = twitter_posts['tweet_text'].apply(vader.polarity_scores).tolist()
      scores_df = pd.DataFrame(scores)
      df = twitter_posts.join(scores_df, rsuffix='_right')
      print(df)
      df['compound'] = df['compound']*(df['scaled_followers_count']+1)
      df.to_csv('tweets.csv')
      return df
  
  # 5. Shared Functions:
  def calc_change_sentiment(data):
      print('Step5a')
      change_in_sent = []
      change_in_sent.append(data['compound'][0])
      for i in range(1,len(data['compound'])):
          if data['compound'][i] == 0:
              change_in_sent.append(0)
          elif data['compound'][i] < 0 or data['compound'][i] > 0:
              dif = data['compound'][i] - data['compound'][(i-1)]
              change_in_sent.append(dif)
      return change_in_sent
  
  def classify_news(dataframe):
      print('Step 5b')
      day1, day2 = [],[]
      for i in range(len(dataframe['timestamp'])):
          if dataframe['timestamp'][i].day == dataframe['timestamp'][i].day and (dataframe['timestamp'][i].hour <= 15 and dataframe['timestamp'][i].hour >= 9):
              day1.append(i)
          elif dataframe['timestamp'][i].day == dataframe['timestamp'][i].day+1 and (dataframe['timestamp'][i].hour <= 15 and dataframe['timestamp'][i].hour >= 9):
              day2.append(i)
          else:
              pass
      news_d1, news_d2 = dataframe.iloc[day1],dataframe.iloc[day2]
      return news_d1, news_d2
  
  # 6. Preprocess Tweets:
  def preprocess_posts(df):
      print('Step 6')
      df.drop(['neg','neu','pos','followers_count'],axis=1,inplace=True)
      df['timestamp'] = df['timestamp'].dt.tz_convert('America/Montreal').dt.tz_localize(None)
      #.dt.tz_localize('UTC').dt.tz_convert('America/Montreal').dt.tz_localize(None)
      df.set_index('timestamp', inplace=True)
      twitter_df_30m = df.resample('30min').median().ffill().reset_index()
      change_in_sent = calc_change_sentiment(twitter_df_30m)
      twitter_sma = twitter_df_30m['compound'].rolling(3).mean()
      twitter_df_30m['Compound SMA(3) Twitter'] = twitter_sma
      twitter_df_30m['change in sentiment twitter'] = change_in_sent
      twitter_df_30m['change in sentiment twitter (t-1)'] = twitter_df_30m['change in sentiment twitter'].shift(1)
      tweet_d1,tweet_d2 = classify_news(twitter_df_30m)
      tweet_d1_red,tweet_d2_red= tweet_d1.iloc[1:],tweet_d2.iloc[1:]
      frames = [tweet_d1_red,tweet_d2_red]
      processed_tweets = pd.concat(frames)
      return processed_tweets
  
  # 7. Preprocess Headlines:
  def preprocess_headlines(data):
      print('Step 7')
      data.drop_duplicates(subset='headline',keep=False, inplace=True)
      data.drop(['ticker','neg','neu','pos'], axis=1, inplace=True)
      data.set_index('timestamp', inplace=True)
      data_30m = data.resample('30min').median().ffill().reset_index()
      change_in_sent=calc_change_sentiment(data_30m)
      headline_sma = data_30m['compound'].rolling(3).mean()
      data_30m['Compound SMA(3) Headlines'] = headline_sma
      data_30m['change in sentiment headlines'] = change_in_sent
      data_30m['change in sentiment headlines (t-1)'] = data_30m['change in sentiment headlines'].shift(1)
      news_d1, news_d2 = classify_news(data_30m)
      news_d1_red, news_d2_red = news_d1.iloc[1:],news_d2.iloc[1:]
      frames_news = [news_d1_red, news_d2_red]
      processed_headlines = pd.concat(frames_news)
      return processed_headlines
  
  # 8. Retrieve, Process, and Merge:
  def data_merge(ticker):
      print('Step 8')
      stock_data = get_stock(ticker)
      headlines = get_news(ticker)
      tweets = get_tweets(ticker)
      print('Collection done..')
      processed_headlines = preprocess_headlines(headlines)
      processed_tweets = preprocess_posts(tweets)
      print('processing....')
      print(tweets)
      print(processed_headlines)
      with_twitter_df = stock_data.merge(processed_tweets, left_on='Datetime', right_on='timestamp',how='left').ffill().drop('timestamp',axis=1)
      full_df = with_twitter_df.merge(processed_headlines, left_on='Datetime', right_on='timestamp',how='left').ffill().drop('timestamp',axis=1)
  
      #full_df = tweets.merge(processed_headlines, left_on='Datetime', right_on='timestamp',how='left').ffill().drop('timestamp',axis=1)
  
      full_df['Percent Price Change Within Period (t+1)'] = full_df['Percent Price Change Within Period'].shift(-1)
      return full_df
  
  # 9. Import Model and Predict:
  def make_prediction(ticker):
      print('Step 9')
      dataframe = data_merge(ticker)
      x_var = ['Adj Close','Scaled Volume','compound_y','compound_x','Compound SMA(3) Headlines','Compound SMA(3) Twitter','SMA(3)','change in sentiment headlines','change in sentiment headlines (t-1)','change in sentiment twitter','change in sentiment twitter (t-1)']
      print('Here.')
      X_test = dataframe[x_var][-2:]
      loaded_model = pickle.load(open('finalized_xgb_model.sav', 'rb'))
      result = ((loaded_model.predict(X_test)/100) * dataframe['Adj Close'].iloc[-2]) + dataframe['Adj Close'].iloc[-2]
      previous_price = dataframe['Adj Close'].iloc[-2]
      print(result)
      print('Result found!')
      if result[0] > previous_price:
          print('The predicted close for {} is ${}, up from ${}.'.format(ticker, "%.2f" % result[0],"%.2f" % previous_price))
          return result
         # return (ticker.upper(),'The predicted close for {} is ${}, up from ${}.'.format(ticker, "%.2f" % result[0],"%.2f" % previous_price))
      else:
          print('The predicted close for {} is ${}, down from ${}.'.format(ticker, "%.2f" % result[0],"%.2f" % previous_price))
          return result
          #return (ticker.upper(),'The predicted close for {} is ${}, down from ${}.'.format(ticker, "%.2f" % result[0],"%.2f" % previous_price))
  
  # pickle. load(open(filename, 'rb'))
  import nltk
  nltk.download('vader_lexicon')
  
  #10. Execute:
  if __name__ == '__main__':
      # chosen_stocks = pd.read_csv('stock_tickers.csv', sep=r'\s*,\s*', header=0)
      # tickers = chosen_stocks['ticker'].tolist()
      # for ticker in tickers:
      #     make_prediction(ticker)
  
      ticker = 'NVAX'
      print(make_prediction(ticker))
  
  