text: # -*- coding: utf-8 -*-
  """NewsSentiment.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1abfT7VJ4iSrQ38urJdgawlHz79Y4wi4N
  """
  
  # Commented out IPython magic to ensure Python compatibility.
  import pandas as pd
  import numpy as np
  from scipy.sparse import hstack, csr_matrix
  from textblob import TextBlob
  import nltk
  import re
  import datetime
  import matplotlib.pyplot as plt
  from nltk.corpus import stopwords 
  from nltk.tokenize import word_tokenize
  from nltk.stem import WordNetLemmatizer, PorterStemmer
  from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
  from sklearn.preprocessing import LabelEncoder, StandardScaler
  from sklearn.model_selection import train_test_split
  from sklearn.decomposition import TruncatedSVD
  from sklearn.svm import LinearSVR
  import lightgbm as lgb
  from sklearn.ensemble import RandomForestRegressor
  from sklearn.metrics import mean_absolute_error
  
  # %matplotlib inline
  
  """Read data files"""
  
  train = pd.read_csv('/content/train_file.csv')
  test = pd.read_csv('/content/test_file.csv')
  submission = pd.read_csv('/content/sample_submission.csv')
  test_id = test['IDLink']
  
  """Explore data"""
  
  train.head()
  
  train.info()
  
  train.describe()
  
  """Check for missing values"""
  
  train.isnull().sum()
  
  test.isnull().sum()
  
  train['Source'].value_counts()[:6
                              ]
  
  train['Source'] = train['Source'].fillna('Bloomberg')
  test['Source'] = test['Source'].fillna('Bloomberg')
  
  """Text columns processing and cleaning"""
  
  nltk.download('stopwords')
  nltk.download('punkt')
  nltk.download('wordnet')
  
  stop = set(stopwords.words('english'))
  
  def clean(text):
    text_token = word_tokenize(text)
    filtered_text = ' '.join([w.lower() for w in text_token if w.lower() not in stop and len(w) > 2])
    filtered_text = filtered_text.replace(r"[^a-zA-Z]+", '')
    text_only = re.sub(r'\b\d+\b', '', filtered_text)
    clean_text = text_only.replace(',', '').replace('.', '').replace(':', '')
    return clean_text
  
  train['Text_Title'] = train['Title'] + ' ' + train['Source'] + ' ' + train['Topic']
  test['Text_Title'] = test['Title'] + ' ' + test['Source'] + ' ' + test['Topic']
  
  train['Text_Headline'] = train['Headline'] + ' ' + train['Source'] + ' ' + train['Topic']
  test['Text_Headline'] = test['Headline'] + ' ' + test['Source'] + ' ' + test['Topic']
  
  train['Text_Title'][6]
  
  train['Text_Title'] = [clean(x) for x in train['Text_Title']]
  test['Text_Title'] = [clean(x) for x in test['Text_Title']]
  
  train['Text_Headline'] = [clean(x) for x in train['Text_Headline']]
  test['Text_Headline'] = [clean(x) for x in test['Text_Headline']]
  
  train['Text_Title'][6]
  
  text = ""
  for ind, row in train.iterrows():
      text += row["Text_Title"] + " "
  text = text.strip()
  
  text = ""
  for ind, row in train.iterrows():
      text += row["Text_Headline"] + " "
  text = text.strip()
  
  """Feature extraction from text data"""
  
  vectorizer = TfidfVectorizer(use_idf=True)
  
  train_v_Title = vectorizer.fit_transform(train['Text_Title'])
  test_v_Title = vectorizer.transform(test['Text_Title'])
  
  vectorizer_ = TfidfVectorizer()
  
  train_v_Headline = vectorizer_.fit_transform(train['Text_Headline'])
  test_v_Headline = vectorizer_.transform(test['Text_Headline'])
  
  """SVD to reduce dimensions"""
  
  # svd = TruncatedSVD(n_components=20)
  
  # train_v_Title = svd.fit_transform(train_v_Title)
  # test_v_Title = svd.transform(test_v_Title)
  
  # train_v_Headline = svd.fit_transform(train_v_Headline)
  # test_v_Headline = svd.transform(test_v_Headline)
  
  """Calculate sentiment of text data"""
  
  train['polarity_t'] = train['Title'].apply(lambda x: TextBlob(x).sentiment.polarity)
  test['polarity_t'] = test['Title'].apply(lambda x: TextBlob(x).sentiment.polarity)
  
  train['subjectivity_t'] = train['Title'].apply(lambda x: TextBlob(x).sentiment.subjectivity)
  test['subjectivity_t'] = test['Title'].apply(lambda x: TextBlob(x).sentiment.subjectivity)
  
  train['polarity_h'] = train['Headline'].apply(lambda x: TextBlob(x).sentiment.polarity)
  test['polarity_h'] = test['Headline'].apply(lambda x: TextBlob(x).sentiment.polarity)
  
  train['subjectivity_h'] = train['Headline'].apply(lambda x: TextBlob(x).sentiment.subjectivity)
  test['subjectivity_h'] = test['Headline'].apply(lambda x: TextBlob(x).sentiment.subjectivity)
  
  """Encode categorical columns"""
  
  encoder = LabelEncoder()
  
  train['Topic'] = encoder.fit_transform(train['Topic'])
  test['Topic'] = encoder.transform(test['Topic'])
  
  total = train['Source'].to_list() + test['Source'].to_list()
  total = encoder.fit_transform(total)
  train['Source'] = encoder.transform(train['Source'])
  test['Source'] = encoder.transform(test['Source'])
  
  """Meta Features"""
  
  # Get day-type(monday, tuesday) from datetime
  
  train_weekday = []
  test_weekday = []
  
  for i in train['PublishDate']:
      train_weekday.append(datetime.datetime.strptime(i, "%Y-%m-%d %H:%M:%S").strftime("%A"))
      
  for i in test['PublishDate']:
      test_weekday.append(datetime.datetime.strptime(i, "%Y-%m-%d %H:%M:%S").strftime("%A"))
  
  train['weekday'] = train_weekday
  test['weekday'] = test_weekday
  
  
  # convert weekday to 0-6
  
  train['weekday'] = train['weekday'].map({'Monday': 0,
                                          'Tuesday': 1,
                                          'Wednesday': 2,
                                          'Thursday': 3,
                                          'Friday': 4,
                                          'Saturday': 5,
                                          'Sunday': 6})
  test['weekday'] = test['weekday'].map({'Monday': 0,
                                          'Tuesday': 1,
                                          'Wednesday': 2,
                                          'Thursday': 3,
                                          'Friday': 4,
                                          'Saturday': 5,
                                          'Sunday': 6})
  
  # Hour from date
  
  train["hour"] = train["PublishDate"].apply(lambda x: x.split()[1].split(':')[0])
  test["hour"] = test["PublishDate"].apply(lambda x: x.split()[1].split(':')[0])
  
  # Number of words in the Title 
  train["num_words_t"] = train["Text_Title"].apply(lambda x: len(str(x).split()))
  test["num_words_t"] = test["Text_Title"].apply(lambda x: len(str(x).split()))
  
  # Number of unique words in the Title 
  train["num_unique_words_t"] = train["Text_Title"].apply(lambda x: len(set(str(x).split())))
  test["num_unique_words_t"] = test["Text_Title"].apply(lambda x: len(set(str(x).split())))
  
  # Number of characters in the Title 
  train["num_chars_t"] = train["Text_Title"].apply(lambda x: len(str(x)))
  test["num_chars_t"] = test["Text_Title"].apply(lambda x: len(str(x)))
  
  # Average length of the words in the Title 
  train["mean_word_len_t"] = train["Text_Title"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
  test["mean_word_len_t"] = test["Text_Title"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
  
  # Number of words in the Headline 
  train["num_words_h"] = train["Text_Headline"].apply(lambda x: len(str(x).split()))
  test["num_words_h"] = test["Text_Headline"].apply(lambda x: len(str(x).split()))
  
  # Number of unique words in the Headline 
  train["num_unique_words_h"] = train["Text_Headline"].apply(lambda x: len(set(str(x).split())))
  test["num_unique_words_h"] = test["Text_Headline"].apply(lambda x: len(set(str(x).split())))
  
  # Number of characters in the Headline 
  train["num_chars_h"] = train["Text_Headline"].apply(lambda x: len(str(x)))
  test["num_chars_h"] = test["Text_Headline"].apply(lambda x: len(str(x)))
  
  # Average length of the words in the Headline 
  train["mean_word_len_h"] = train["Text_Headline"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
  test["mean_word_len_h"] = test["Text_Headline"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
  
  scaler = StandardScaler()
  
  cols = ['Source', 'Topic', 'Facebook', 'GooglePlus', 'LinkedIn', 'num_words_t', 'num_unique_words_t', 'num_chars_t', 'mean_word_len_t',
          'num_words_h', 'num_unique_words_h', 'num_chars_h', 'mean_word_len_h', 'hour', 'weekday']
  
  for col in cols:
    train[col] = scaler.fit_transform(train[col].values.reshape(-1, 1))
    test[col] = scaler.transform(test[col].values.reshape(-1, 1))
  
  cols_t = ['Source', 'Topic', 'Facebook', 'GooglePlus', 'LinkedIn', 'num_words_t', 'num_unique_words_t', 'num_chars_t', 'mean_word_len_t', 'polarity_t', 'subjectivity_t', 'hour', 'weekday']
  train_X1 = train[cols_t]
  test_X1 = test[cols_t]
  
  cols_h = ['Source', 'Topic', 'Facebook', 'GooglePlus', 'LinkedIn', 'num_words_h', 'num_unique_words_h', 'num_chars_h', 'mean_word_len_h', 'polarity_h', 'subjectivity_h', 'hour', 'weekday']
  train_X2 = train[cols_h]
  test_X2 = test[cols_h]
  
  train_X1.head()
  
  train_X2.head()
  
  print(np.shape(train_X1))
  print(np.shape(test_X1))
  
  print(np.shape(train_X2))
  print(np.shape(test_X2))
  
  print(np.shape(train_v_Title))
  print(np.shape(test_v_Title))
  
  print(np.shape(train_v_Headline))
  print(np.shape(test_v_Headline))
  
  train_X_Title = hstack([train_v_Title, csr_matrix(train_X1.values)])
  test_X_Title = hstack([test_v_Title, csr_matrix(test_X1.values)])
  y1 = train['SentimentTitle']
  
  train_X_Headline = hstack([train_v_Headline, csr_matrix(train_X2.values)])
  test_X_Headline = hstack([test_v_Headline, csr_matrix(test_X2.values)])
  y2 = train['SentimentHeadline']
  
  np.shape(train_X_Title)
  
  '/device:GPU:0'
  
  """Apply Machine Learning Models"""
  
  # LinearSVR model for SentimentTitle
  
  X_train, X_test, y_train, y_test = train_test_split(train_X_Title, y1, test_size=0.20, random_state=42)
  
  clf1 = RandomForestRegressor(n_estimators=100,
                                  criterion="mae",
                                  n_jobs=-1,
                                  random_state=1)
  clf1.fit(X_train, y_train)
  
  y_pred1 = clf1.predict(X_test)
  mae1 = mean_absolute_error(y_pred1, y_test)
  print('MAE:', 1 - mae1)
  
  # LinearSVR model for SentimentHeadline
  
  X_train, X_test, y_train, y_test = train_test_split(train_X_Headline, y2, test_size=0.20, random_state=42)
  
  clf2 = RandomForestRegressor(n_estimators=100,
                                  criterion="mae",
                                  n_jobs=-1,
                                  random_state=1)
  clf2.fit(X_train, y_train)
  
  y_pred2 = clf2.predict(X_test)
  mae2 = mean_absolute_error(y_pred2, y_test)
  print('MAE:', 1 - mae2)
  
  print('MAE:', 1 - ((0.4 * mae1) + (0.6 * mae2)))
  
  """Make predictions for test dataset"""
  
  title = clf1.predict(test_X_Title)
  headline = clf2.predict(test_X_Headline)
  
  """Save results in csv"""
  
  df = pd.DataFrame()
  df['IDLink'] = test_id
  df['SentimentTitle'] = title
  df['SentimentHeadline'] = headline
  df.to_csv('/content/submit1.csv', index=False)
  
  
  
  