# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="S0KeVwv61nYC"
import gensim

# + colab={"base_uri": "https://localhost:8080/"} id="UXZbyy1i14bf" outputId="cb8c944b-78e0-4a5b-a453-16da6dadd32c"
import numpy as np
import pandas as pd
import os
import xml.etree.ElementTree as ET
import nltk
nltk.download('punkt')
from nltk.tokenize import sent_tokenize, word_tokenize

# + colab={"base_uri": "https://localhost:8080/"} id="IHVknudtWwWs" outputId="59ddba22-2dcb-424b-f95e-b8f7fb028230"
from google.colab import drive
drive.mount('/content/drive')

# + colab={"base_uri": "https://localhost:8080/"} id="DyUIgtQG2P1D" outputId="9ee2972f-b829-480d-f661-921b623c9ed0"
os.chdir("/content/drive/MyDrive")
xml_data = open('./dataset/articles-training-byarticle-20181122.xml', 'r').read()  # Read file
root = ET.XML(xml_data)

data = []

for i, child in enumerate(root):
    data.append([subchild.text for subchild in child])

df = pd.DataFrame(data).T  # Write in DF and transpose it

print(data[218])

# + colab={"base_uri": "https://localhost:8080/"} id="TxNVwrLt2Seg" outputId="1f2488ce-b8c3-43d7-95f1-0593d2bf64fd"
xml_data1 = open('./dataset/ground-truth-training-byarticle-20181122.xml', 'r').read()  # Read file
root1 = ET.XML(xml_data1)

labels = []

for i, child in enumerate(root1):
    if child.attrib['hyperpartisan']=='true':
        labels.append(1)
    else:
        labels.append(0)

print(labels)

# + id="NZBYoIy32W68"
training_data = data[0:500]
training_labels = labels[0:500]

testing_data = data[500:640]
testing_labels = labels[500:640]

# + id="E5wjQhr-3FFm"
train_X = []
train_Y = []

sum = 0
for i in range(len(training_data)):
  sentences = []
  for j in range(len(training_data[i])):

    if training_data[i][j] is None:
      continue
    if(len(training_data[i][j])<3):
      continue

    # print(data[i][j])
    lst = sent_tokenize(training_data[i][j])

    for k in range(len(lst)):
      sentences.append(lst[k])

  tokenized_context_send = []

  for j in range(len(sentences)):

    raw_context = sentences[j]
    tokenized_context1 = raw_context.split()

    new_list = []
    for word in tokenized_context1:
      word1 = word.lower()

      new_word = ""

      for p in word1:
        if (p.isalpha()):
          new_word+=p

      if(len(new_word)>2 and new_word!="the"):
        new_list.append(new_word)

    if(len(new_list)>1):
      tokenized_context_send.append(new_list)

  # print(tokenized_context_send)
  
  train_X.append(tokenized_context_send)
  train_Y.append(training_labels[i])


# + id="onxvO21Ae30J"
test_X = []
test_Y = []

sum = 0
for i in range(len(testing_data)):
  sentences = []
  for j in range(len(testing_data[i])):

    if testing_data[i][j] is None:
      continue
    if(len(testing_data[i][j])<3):
      continue

    # print(data[i][j])
    lst = sent_tokenize(testing_data[i][j])

    for k in range(len(lst)):
      sentences.append(lst[k])

  tokenized_context_send = []

  for j in range(len(sentences)):

    raw_context = sentences[j]
    tokenized_context1 = raw_context.split()

    new_list = []
    for word in tokenized_context1:
      word1 = word.lower()

      new_word = ""

      for p in word1:
        if (p.isalpha()):
          new_word+=p

      if(len(new_word)>2 and new_word!="the"):
        new_list.append(new_word)

    if(len(new_list)>1):
      tokenized_context_send.append(new_list)

  # print(tokenized_context_send)
  
  test_X.append(tokenized_context_send)
  test_Y.append(testing_labels[i])

# + id="-_Q37j5fXBbA"
sentences = []
for i in range(len(train_X)):
  for j in range(len(train_X[i])):
    sentences.append(train_X[i][j])

for i in range(len(test_X)):
  for j in range(len(test_X[i])):
    sentences.append(test_X[i][j])

# print(sentences[0:10])

# + colab={"base_uri": "https://localhost:8080/"} id="Wlu7Uy7Z1v92" outputId="b8e0d120-35ec-496b-b7e9-0b4136126e7d"
word_model = gensim.models.Word2Vec(sentences, size=100 , min_count=1 , window=5, iter=100)
pretrained_weights = word_model.wv.syn0
vocab_size, emdedding_size = pretrained_weights.shape
print('Result embedding shape:', pretrained_weights.shape)
print('Checking similar words:')
for word in ['model', 'network', 'train', 'learn']:
  most_similar = ', '.join('%s (%.2f)' % (similar, dist) 
                           for similar, dist in word_model.most_similar(word)[:8])
  print('  %s -> %s' % (word, most_similar))

def word2idx(word):
  return word_model.wv.vocab[word].index
def idx2word(idx):
  return word_model.wv.index2word[idx]


# + id="I9JBQzgifa4H"
print(pretrained_weights[0])

# + colab={"base_uri": "https://localhost:8080/"} id="qBMqXY8Ofv0S" outputId="9d94ce1a-7972-4389-9241-74064b3b7cae"
final_train_X = []
final_train_Y = []
for i in range(len(train_X)):
  feature_vector = []
  for j in range(len(train_X[i])):
    
    sm = 0
    length = len(train_X[i][j])
    for words in train_X[i][j]:
      weight = pretrained_weights[word2idx(words)]
    
      sm1 = 0
      for k in range(len(weight)):
        sm1=sm1+weight[k]

      sm = sm + sm1/len(weight)

    feature_vector.append(sm)

  print(feature_vector)
  final_train_X.append(feature_vector)
  final_train_Y.append(train_Y[i])

# + id="mjFH2NpLllnP"
final_test_X = []
final_test_Y = []
for i in range(len(test_X)):
  feature_vector = []
  for j in range(len(test_X[i])):
    
    sm = 0
    length = len(test_X[i][j])
    for words in test_X[i][j]:
      weight = pretrained_weights[word2idx(words)]
    
      sm1 = 0
      for k in range(len(weight)):
        sm1=sm1+weight[k]

      sm = sm + sm1/len(weight)

    feature_vector.append(sm)

  final_test_X.append(feature_vector)
  final_test_Y.append(test_Y[i])

# + colab={"base_uri": "https://localhost:8080/"} id="0qzuNPilmFvq" outputId="c4e90af1-f93a-4a86-ea30-dd0968bc8024"

# print(np.array(final_train_X).shape)
print(len(final_train_X[0]))
print(final_train_Y)

# + id="str7dvy5ZBji"
from keras.preprocessing.sequence import pad_sequences

train_Xp = pad_sequences( final_train_X , maxlen=100 , dtype='float32')
test_Xp = pad_sequences( final_test_X , maxlen=100 , dtype='float32')


# + id="kRkd0-0EmmmY"
from keras import Sequential
from keras.layers import Embedding, LSTM, Dense, Dropout , Activation , Bidirectional
from keras.layers import SimpleRNN


# + id="DAzrGLms7Qqv"
def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    return np.exp(x) / np.sum(np.exp(x), axis=0)

for i in range(len(train_Xp)):
  train_Xp[i] = softmax(train_Xp[i])

for i in range(len(test_Xp)):
  test_Xp[i] = softmax(test_Xp[i])

# + id="rZL2KNqq1wyy"
model = Sequential()


model.add(Embedding(input_dim=vocab_size,
                    output_dim=emdedding_size,
                    weights=[pretrained_weights],
                    input_length=100,
                    mask_zero=True,
                    trainable=False))

model.add(Bidirectional(LSTM(100)))
model.add(Dense(2, activation='softmax'))

model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

train_X1 = np.asarray(train_Xp)
train_Y1 = np.asarray(final_train_Y)

print(train_X1)
print(train_Y1)

history = model.fit(train_X1, train_Y1, epochs=5, batch_size=64, verbose=1)

# + colab={"base_uri": "https://localhost:8080/"} id="v8py1N9hm_1R" outputId="021ee699-b264-4dbc-9636-5b9e0ef676c0"
test_X1 = np.asarray(test_Xp)
test_Y1 = np.asarray(final_test_Y)


scores = model.evaluate(test_X1, test_Y1, verbose=0)
print(scores[1])

# + id="6OTKGGGjO6rN"
from sklearn.svm import SVC
model1 = SVC(C=0.1 , gamma='scale' , kernel='rbf')

model1.fit(train_X1,train_Y1)
y1_predict = model1.predict(test_X1)


# + colab={"base_uri": "https://localhost:8080/"} id="KQi-2zfDPa9A" outputId="6d108173-fec4-4a19-ee89-8be32e4ec8d7"

print(classification_report(test_Y1, y1_predict))
print('-----------------------------------------------------------')
print(f"Accuracy: {round(accuracy_score(test_Y1, y1_predict), 2)}") 

StarcoderdataJupyterScriptsDedupFiltered