# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="guuhSs1qXxN-" colab_type="code" colab={}
import nltk
import gensim
import pandas as pd

import datetime

# + id="T-x0n9Cr2kf7" colab_type="code" outputId="1fecb4e4-6e9d-40fc-de1a-21139ab3a540" colab={"base_uri": "https://localhost:8080/", "height": 88}
import spacy
# !python -m spacy download en_core_web_lg

# + id="sTpx0bDpBVmL" colab_type="code" colab={}
nlp = spacy.load("en_core_web_lg")

# + id="vL0AmlCe8Ihp" colab_type="code" colab={}
"""Gives you a size 300 array of all the relevant entries."""
def aggregate_score(all_relevant_entries):
  sum = [0]*300
  for i in all_relevant_entries.values:
    # Note: I'm using SpaCy's vectorizer here, but transition to Google's thing
    # if this doesn't work at all
    doc = nlp(i[0])
    sum = sum + doc.vector * i[1]
  return sum


# + [markdown] id="7HRTqFFBdB86" colab_type="text"
# # Dealing with the Reddit data

# + id="y04hw8MxdGMt" colab_type="code" colab={}
red = pd.read_csv("reddits.csv", index_col=0)


# + id="EGIj3Gxyj5Eo" colab_type="code" colab={}
def utc_to_date(utc):
  return datetime.datetime.utcfromtimestamp(utc).strftime('%Y-%m-%d')


# + id="SIJlLTF1dKQZ" colab_type="code" outputId="4fe6c069-0548-4933-f6d6-aa93719fc6da" colab={"base_uri": "https://localhost:8080/", "height": 204}
red.head()

# + id="UXXEqCnwdLFI" colab_type="code" colab={}
red2 = red.copy()
red2["date"] = red2.created_utc.apply(utc_to_date)

# + id="RMgRk3DxkA-c" colab_type="code" colab={}
aggs = red2.groupby("date").apply(aggregate_score)

# + id="wtSazab2kwAk" colab_type="code" outputId="8b763adf-3d80-40b8-88b1-4bf676707a46" colab={"base_uri": "https://localhost:8080/", "height": 238}
aggs

# + id="1cww2UHclR4d" colab_type="code" outputId="cf47078b-79d1-4b84-8f8a-3809ff7a01f1" colab={"base_uri": "https://localhost:8080/", "height": 419}
final_vectors = pd.DataFrame(data={"vector": aggs}, index=red2.date.unique())
final_vectors

# + id="x9RHIkWjog9j" colab_type="code" colab={}
stock_market = pd.read_csv("stock_market.csv", index_col=0)

# + id="1pkJNJrsq-tY" colab_type="code" outputId="e18b8e43-c424-472e-b8a5-c6f42d32c0ab" colab={"base_uri": "https://localhost:8080/", "height": 450}
sm = stock_market.drop(columns=["year", "month", "day", "utc"])
sm

# + id="-2AoCIoKrB58" colab_type="code" outputId="662cc8e9-37ed-42ad-eead-ddee2b965d56" colab={"base_uri": "https://localhost:8080/", "height": 450}
sm = sm.rename(columns={"1. open": "open", "4. close": "close"})
sm["open"] = sm["open"].shift(1)
sm.dropna(axis=0, inplace=True)
sm

# + id="cxNaMD3avzHO" colab_type="code" outputId="429f3fa4-592d-4c52-87b4-1b78b7744ea5" colab={"base_uri": "https://localhost:8080/", "height": 450}
newsm = sm.join(final_vectors, on="index", how="inner")
newsm = newsm.rename(columns={"1. open": "open", "4. close": "close"})
newsm
# newsm.vector['2020-02-10']

# + id="Dut9QyvLrKDH" colab_type="code" colab={}
all_data = newsm
all_data.to_csv("all_data.csv")

# + id="GJbj36xpre0n" colab_type="code" outputId="7c08c184-45ac-4a47-b452-f45389348328" colab={"base_uri": "https://localhost:8080/", "height": 450}
all_data

# + [markdown] id="Ovp_gaUGxnPZ" colab_type="text"
# # Making a Model

# + id="_8wWFfv-4EOM" colab_type="code" colab={}
import pickle

# + id="tjgC_udF7hhe" colab_type="code" colab={}
with open('data.pkl', 'wb') as output:
  pickle.dump(all_data, output)

# + id="SFOBWQ6g7rm_" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered